chore: prepare tokio-util 0.7.4 (#4987)

Years ago, Tokio was organized as a cluster of separate crates. This
architecture required a trait to represent "park the thread and do
work.". When all crates were combined into a single crate, the `Park`
and `Unpark` traits remained as internal details.

This patch removes these traits as they are no longer needed. This is in
service of a future refactor that will decouple the various resource
drivers and store them in a single struct instead of nesting them.
However, in the mean time, removing the Park/Unpark traits adds a bit of
messy code to make the conditional compilation work. This code will
(hopefully) be short-lived.

.circleci/config.yml

@@ -5,7 +5,7 @@ jobs:
       image: ubuntu-2004:202101-01
     resource_class: arm.medium
     environment:
-      # Change to pin rust versino
+      # Change to pin rust version
       RUST_STABLE: stable
     steps:
       - checkout

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,4 @@
+contact_links:
+  - name: Question
+    url: https://github.com/tokio-rs/tokio/discussions
+    about: Questions about Tokio should be posted as a GitHub discussion.

.github/ISSUE_TEMPLATE/question.md

@@ -1,16 +0,0 @@
----
-name: Question
-about: Please use the discussions tab for questions
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Please post your question as a discussion here:
-https://github.com/tokio-rs/tokio/discussions
-
-
-You may also be able to find help here:
-https://discord.gg/tokio
-https://users.rust-lang.org/

.github/workflows/audit.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Audit Check
         uses: actions-rs/audit-check@v1

.github/workflows/ci.yml

@@ -11,8 +11,16 @@ env:
   RUST_BACKTRACE: 1
   # Change to specific Rust release to pin
   rust_stable: stable
-  rust_nightly: nightly-2022-03-21
+  rust_nightly: nightly-2022-07-26
   rust_clippy: 1.52.0
+  # When updating this, also update:
+  # - README.md
+  # - tokio/README.md
+  # - CONTRIBUTING.md
+  # - tokio/Cargo.toml
+  # - tokio-util/Cargo.toml
+  # - tokio-test/Cargo.toml
+  # - tokio-stream/Cargo.toml
   rust_min: 1.49.0
 
 defaults:
@@ -26,20 +34,24 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - test
-      - test-unstable
       - test-parking_lot
+      - valgrind
+      - test-unstable
       - miri
+      - asan
       - cross
       - features
       - minrust
+      - minimal-versions
       - fmt
       - clippy
       - docs
-      - valgrind
       - loom-compile
       - check-readme
       - test-hyper
       - wasm32-unknown-unknown
+      - wasm32-wasi
+      - check-external-types
     steps:
       - run: exit 0
 
@@ -53,7 +65,7 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -103,7 +115,7 @@ jobs:
     name: compile tests with parking lot send guards
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -120,7 +132,7 @@ jobs:
     name: valgrind
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -161,7 +173,7 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -182,7 +194,7 @@ jobs:
     name: miri
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
         uses: actions-rs/toolchain@v1
         with:
@@ -199,11 +211,14 @@ jobs:
           MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-tag-raw-pointers
           PROPTEST_CASES: 10
 
-  san:
-    name: san
+  asan:
+    name: asan
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - name: Install llvm
+        # Required to resolve symbols in sanitizer output
+        run: sudo apt-get install -y llvm
       - name: Install Rust ${{ env.rust_nightly }}
         uses: actions-rs/toolchain@v1
         with:
@@ -211,11 +226,11 @@ jobs:
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: asan
-        run: cargo test --all-features --target x86_64-unknown-linux-gnu --lib -- --test-threads 1
-        working-directory: tokio
+        run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1
         env:
           RUSTFLAGS: -Z sanitizer=address
-          ASAN_OPTIONS: detect_leaks=0
+          # Ignore `trybuild` errors as they are irrelevant and flaky on nightly
+          TRYBUILD: overwrite
 
   cross:
     name: cross
@@ -230,14 +245,13 @@ jobs:
           - arm-linux-androideabi
           - mipsel-unknown-linux-musl
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
           override: true
-      - uses: Swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           use-cross: true
@@ -255,7 +269,7 @@ jobs:
     name: features
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
         uses: actions-rs/toolchain@v1
         with:
@@ -267,6 +281,8 @@ jobs:
         run: cargo install cargo-hack
       - name: check --each-feature
         run: cargo hack check --all --each-feature -Z avoid-dev-deps
+      - name: check net,time
+        run: cargo check -p tokio --no-default-features --features net,time -Z avoid-dev-deps
       # Try with unstable feature flags
       - name: check --each-feature --unstable
         run: cargo hack check --all --each-feature -Z avoid-dev-deps
@@ -277,7 +293,7 @@ jobs:
     name: minrust
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_min }}
         uses: actions-rs/toolchain@v1
         with:
@@ -291,7 +307,7 @@ jobs:
     name: minimal-versions
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
         uses: actions-rs/toolchain@v1
         with:
@@ -323,7 +339,7 @@ jobs:
     name: fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -344,7 +360,7 @@ jobs:
     name: clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_clippy }}
         uses: actions-rs/toolchain@v1
         with:
@@ -360,7 +376,7 @@ jobs:
     name: docs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_nightly }}
         uses: actions-rs/toolchain@v1
         with:
@@ -377,7 +393,7 @@ jobs:
     name: build loom tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -394,7 +410,7 @@ jobs:
     name: Check README
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Verify that both READMEs are identical
         run: diff README.md tokio/README.md
 
@@ -412,7 +428,7 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -440,7 +456,7 @@ jobs:
     name: test tokio for wasm32-unknown-unknown
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:
@@ -452,3 +468,75 @@ jobs:
       - name: test tokio
         run: wasm-pack test --node -- --features "macros sync"
         working-directory: tokio
+
+  wasm32-wasi:
+    name: wasm32-wasi
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          override: true
+      - uses: Swatinem/rust-cache@v1
+
+      # Install dependencies
+      - name: Install cargo-hack
+        run: cargo install cargo-hack
+
+      - name: Install wasm32-wasi target
+        run: rustup target add wasm32-wasi
+
+      - name: Install wasmtime
+        run: cargo install wasmtime-cli
+
+      - name: Install cargo-wasi
+        run: cargo install cargo-wasi
+
+      - name: WASI test tokio full
+        run: cargo test -p tokio --target wasm32-wasi --features full
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+      - name: WASI test tokio-util full
+        run: cargo test -p tokio-util --target wasm32-wasi --features full
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+      - name: WASI test tokio-stream
+        run: cargo test -p tokio-stream --target wasm32-wasi --features time,net,io-util,sync
+        env:
+          CARGO_TARGET_WASM32_WASI_RUNNER: "wasmtime run --"
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+
+      - name: test tests-integration --features wasi-rt
+        # TODO: this should become: `cargo hack wasi test --each-feature`
+        run: cargo wasi test --test rt_yield --features wasi-rt
+        working-directory: tests-integration
+
+  check-external-types:
+    name: check-external-types
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_nightly }}
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - name: check-external-types
+        run: |
+          set -x
+          cargo install cargo-check-external-types --locked --version 0.1.3
+          cargo check-external-types --all-features --config external-types.toml
+        working-directory: tokio
+

.github/workflows/labeler.yml

@@ -7,6 +7,7 @@ on:
 jobs:
   triage:
     runs-on: ubuntu-latest
+    if: github.repository_owner == 'tokio-rs'
     steps:
     - uses: actions/labeler@v3
       with:

.github/workflows/loom.yml

@@ -17,7 +17,7 @@ jobs:
   loom:
     name: loom
     # base_ref is null when it's not a pull request
-    if: contains(github.event.pull_request.labels.*.name, 'R-loom') || (github.base_ref == null)
+    if: github.repository_owner == 'tokio-rs' && (contains(github.event.pull_request.labels.*.name, 'R-loom') || (github.base_ref == null))
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -29,7 +29,7 @@ jobs:
           - loom_pool::group_d
           - time::driver
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:

.github/workflows/pr-audit.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install cargo-audit
         uses: actions-rs/cargo@v1

.github/workflows/stress-test.yml

@@ -12,7 +12,7 @@ env:
   rust_stable: stable
 
 jobs:
-  stess-test:
+  stress-test:
     name: Stress Test
     runs-on: ubuntu-latest
     strategy:
@@ -20,7 +20,7 @@ jobs:
         stress-test:
           - simple_echo_tcp
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust ${{ env.rust_stable }}
         uses: actions-rs/toolchain@v1
         with:

CONTRIBUTING.md

@@ -131,12 +131,30 @@ cargo check --all-features
 cargo test --all-features
 ```
 
+Clippy must be run using the MSRV, so Tokio can avoid having to `#[allow]` new
+lints whose fixes would be incompatible with the current MSRV:
+
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
+```
+cargo +1.49.0 clippy --all --tests --all-features
+```
+
 When building documentation normally, the markers that list the features
 required for various parts of Tokio are missing. To build the documentation
 correctly, use this command:
 
 ```
-RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features
+RUSTDOCFLAGS="--cfg docsrs" RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features
 ```
 
 To build documentation including Tokio's unstable features, it is necessary to
@@ -144,15 +162,9 @@ pass `--cfg tokio_unstable` to both RustDoc *and* rustc. To build the
 documentation for unstable features, use this command:
 
 ```
-RUSTDOCFLAGS="--cfg docsrs --cfg tokio_unstable" RUSTFLAGS="--cfg tokio_unstable" cargo +nightly doc --all-features
+RUSTDOCFLAGS="--cfg docsrs --cfg tokio_unstable" RUSTFLAGS="--cfg docsrs --cfg tokio_unstable" cargo +nightly doc --all-features
 ```
 
-There is currently a [bug in cargo] that means documentation cannot be built
-from the root of the workspace. If you `cd` into the `tokio` subdirectory the
-command shown above will work.
-
-[bug in cargo]: https://github.com/rust-lang/cargo/issues/9274
-
 The `cargo fmt` command does not work on the Tokio codebase. You can use the
 command below instead:
 
@@ -544,7 +556,7 @@ Tokio ≥1.0.0 comes with LTS guarantees:
 
 The goal of these guarantees is to provide stability to the ecosystem.
 
-## Mininum Supported Rust Version (MSRV)
+## Minimum Supported Rust Version (MSRV)
 
  * All Tokio ≥1.0.0 releases will support at least a 6-month old Rust
    compiler release.

README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.18.2", features = ["full"] }
+tokio = { version = "1.21.0", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -161,8 +161,30 @@ several other libraries, including:
 [`mio`]: https://github.com/tokio-rs/mio
 [`bytes`]: https://github.com/tokio-rs/bytes
 
+## Changelog
+
+The Tokio repository contains multiple crates. Each crate has its own changelog.
+
+ * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md)
+ * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md)
+ * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md)
+ * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md)
+ * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md)
+
 ## Supported Rust Versions
 
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- CONTRIBUTING.md
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
 Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
 least** 6 months. When increasing the MSRV, the new Rust version must have been
 released at least six months ago. The current MSRV is 1.49.0.
@@ -180,18 +202,18 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.8.x` - LTS release until February 2022.
- * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until June 2023
+ * `1.20.x` - LTS release until September 2023.
 
-Each LTS release will continue to receive backported fixes for at least half a
-year. If you wish to use a fixed minor release in your project, we recommend
-that you use an LTS release.
+Each LTS release will continue to receive backported fixes for at least a year.
+If you wish to use a fixed minor release in your project, we recommend that you
+use an LTS release.
 
 To use a fixed minor version, you can specify the version with a tilde. For
-example, to specify that you wish to use the newest `1.8.x` patch release, you
+example, to specify that you wish to use the newest `1.18.x` patch release, you
 can use the following dependency specification:
 ```text
-tokio = { version = "~1.8", features = [...] }
+tokio = { version = "~1.18", features = [...] }
 ```
 
 ## License

SECURITY.md

@@ -1,13 +1,13 @@
 ## Report a security issue
 
-The Tokio project team welcomes security reports and is committed to providing prompt attention to security issues. Security issues should be reported privately via [[email protected]](mailto:[email protected]). Security issues should not be reported via the public Github Issue tracker.
+The Tokio project team welcomes security reports and is committed to providing prompt attention to security issues. Security issues should be reported privately via [[email protected]](mailto:[email protected]). Security issues should not be reported via the public GitHub Issue tracker.
 
 ## Vulnerability coordination
 
-Remediation of security vulnerabilities is prioritized by the project team. The project team coordinates remediation with third-party project stakeholders via [Github Security Advisories](https://help.github.com/en/github/managing-security-vulnerabilities/about-github-security-advisories). Third-party stakeholders may include the reporter of the issue, affected direct or indirect users of Tokio, and maintainers of upstream dependencies if applicable.
+Remediation of security vulnerabilities is prioritized by the project team. The project team coordinates remediation with third-party project stakeholders via [GitHub Security Advisories](https://help.github.com/en/github/managing-security-vulnerabilities/about-github-security-advisories). Third-party stakeholders may include the reporter of the issue, affected direct or indirect users of Tokio, and maintainers of upstream dependencies if applicable.
 
-Downstream project maintainers and Tokio users can request participation in coordination of applicable security issues by sending your contact email address, Github username(s) and any other salient information to [[email protected]](mailto:[email protected]). Participation in security issue coordination processes is at the discretion of the Tokio team.
+Downstream project maintainers and Tokio users can request participation in coordination of applicable security issues by sending your contact email address, GitHub username(s) and any other salient information to [[email protected]](mailto:[email protected]). Participation in security issue coordination processes is at the discretion of the Tokio team.
 
 ## Security advisories
 
-The project team is committed to transparency in the security issue disclosure process. The Tokio team announces security issues via [project Github Release notes](https://github.com/tokio-rs/tokio/releases) and the [RustSec advisory database](https://github.com/RustSec/advisory-db) (i.e. `cargo-audit`).
+The project team is committed to transparency in the security issue disclosure process. The Tokio team announces security issues via [project GitHub Release notes](https://github.com/tokio-rs/tokio/releases) and the [RustSec advisory database](https://github.com/RustSec/advisory-db) (i.e. `cargo-audit`).

benches/sync_rwlock.rs

@@ -14,7 +14,7 @@ fn read_uncontended(b: &mut Bencher) {
         rt.block_on(async move {
             for _ in 0..6 {
                 let read = lock.read().await;
-                black_box(read);
+                let _read = black_box(read);
             }
         })
     });
@@ -28,7 +28,7 @@ fn read_concurrent_uncontended_multi(b: &mut Bencher) {
 
     async fn task(lock: Arc<RwLock<()>>) {
         let read = lock.read().await;
-        black_box(read);
+        let _read = black_box(read);
     }
 
     let lock = Arc::new(RwLock::new(()));
@@ -55,7 +55,7 @@ fn read_concurrent_uncontended(b: &mut Bencher) {
 
     async fn task(lock: Arc<RwLock<()>>) {
         let read = lock.read().await;
-        black_box(read);
+        let _read = black_box(read);
     }
 
     let lock = Arc::new(RwLock::new(()));
@@ -82,7 +82,7 @@ fn read_concurrent_contended_multi(b: &mut Bencher) {
 
     async fn task(lock: Arc<RwLock<()>>) {
         let read = lock.read().await;
-        black_box(read);
+        let _read = black_box(read);
     }
 
     let lock = Arc::new(RwLock::new(()));
@@ -110,7 +110,7 @@ fn read_concurrent_contended(b: &mut Bencher) {
 
     async fn task(lock: Arc<RwLock<()>>) {
         let read = lock.read().await;
-        black_box(read);
+        let _read = black_box(read);
     }
 
     let lock = Arc::new(RwLock::new(()));

netlify.toml

@@ -0,0 +1,16 @@
+[build]
+  command = """
+    rustup install nightly --profile minimal && cargo doc --no-deps --all-features
+    """
+  publish = "target/doc"
+
+[build.environment]
+  RUSTDOCFLAGS="""
+    --cfg docsrs \
+    --cfg tokio_unstable \
+    """
+  RUSTFLAGS="--cfg tokio_unstable --cfg docsrs"
+
+[[redirects]]
+  from = "/"
+  to = "/tokio"

tests-build/README.md

@@ -1,2 +1,10 @@
 Tests the various combination of feature flags. This is broken out to a separate
 crate to work around limitations with cargo features.
+
+To run all of the tests in this directory, run the following commands:
+```
+cargo test --features full
+cargo test --features rt
+```
+If one of the tests fail, you can pass `TRYBUILD=overwrite` to the `cargo test`
+command that failed to have it regenerate the test output.

tests-build/tests/fail/macros_core_no_default.stderr

@@ -1,5 +1,5 @@
 error: The default runtime flavor is `multi_thread`, but the `rt-multi-thread` feature is disabled.
- --> tests/fail/macros_core_no_default.rs:3:1
+ --> $DIR/macros_core_no_default.rs:3:1
   |
 3 | #[tokio::main]
   | ^^^^^^^^^^^^^^

tests-build/tests/fail/macros_dead_code.stderr

@@ -1,11 +1,11 @@
-error: function is never used: `f`
- --> tests/fail/macros_dead_code.rs:6:10
+error: function `f` is never used
+ --> $DIR/macros_dead_code.rs:6:10
   |
 6 | async fn f() {}
   |          ^
   |
 note: the lint level is defined here
- --> tests/fail/macros_dead_code.rs:1:9
+ --> $DIR/macros_dead_code.rs:1:9
   |
 1 | #![deny(dead_code)]
   |         ^^^^^^^^^

tests-build/tests/fail/macros_invalid_input.stderr

@@ -1,101 +1,101 @@
 error: the `async` keyword is missing from the function declaration
- --> tests/fail/macros_invalid_input.rs:6:1
+ --> $DIR/macros_invalid_input.rs:6:1
   |
 6 | fn main_is_not_async() {}
   | ^^
 
 error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
- --> tests/fail/macros_invalid_input.rs:8:15
+ --> $DIR/macros_invalid_input.rs:8:15
   |
 8 | #[tokio::main(foo)]
   |               ^^^
 
 error: Must have specified ident
-  --> tests/fail/macros_invalid_input.rs:11:15
+  --> $DIR/macros_invalid_input.rs:11:15
    |
 11 | #[tokio::main(threadpool::bar)]
    |               ^^^^^^^^^^^^^^^
 
 error: the `async` keyword is missing from the function declaration
-  --> tests/fail/macros_invalid_input.rs:15:1
+  --> $DIR/macros_invalid_input.rs:15:1
    |
 15 | fn test_is_not_async() {}
    | ^^
 
 error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
-  --> tests/fail/macros_invalid_input.rs:17:15
+  --> $DIR/macros_invalid_input.rs:17:15
    |
 17 | #[tokio::test(foo)]
    |               ^^^
 
 error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
-  --> tests/fail/macros_invalid_input.rs:20:15
+  --> $DIR/macros_invalid_input.rs:20:15
    |
 20 | #[tokio::test(foo = 123)]
    |               ^^^^^^^^^
 
 error: Failed to parse value of `flavor` as string.
-  --> tests/fail/macros_invalid_input.rs:23:24
+  --> $DIR/macros_invalid_input.rs:23:24
    |
 23 | #[tokio::test(flavor = 123)]
    |                        ^^^
 
 error: No such runtime flavor `foo`. The runtime flavors are `current_thread` and `multi_thread`.
-  --> tests/fail/macros_invalid_input.rs:26:24
+  --> $DIR/macros_invalid_input.rs:26:24
    |
 26 | #[tokio::test(flavor = "foo")]
    |                        ^^^^^
 
 error: The `start_paused` option requires the `current_thread` runtime flavor. Use `#[tokio::test(flavor = "current_thread")]`
-  --> tests/fail/macros_invalid_input.rs:29:55
+  --> $DIR/macros_invalid_input.rs:29:55
    |
 29 | #[tokio::test(flavor = "multi_thread", start_paused = false)]
    |                                                       ^^^^^
 
 error: Failed to parse value of `worker_threads` as integer.
-  --> tests/fail/macros_invalid_input.rs:32:57
+  --> $DIR/macros_invalid_input.rs:32:57
    |
 32 | #[tokio::test(flavor = "multi_thread", worker_threads = "foo")]
    |                                                         ^^^^^
 
 error: The `worker_threads` option requires the `multi_thread` runtime flavor. Use `#[tokio::test(flavor = "multi_thread")]`
-  --> tests/fail/macros_invalid_input.rs:35:59
+  --> $DIR/macros_invalid_input.rs:35:59
    |
 35 | #[tokio::test(flavor = "current_thread", worker_threads = 4)]
    |                                                           ^
 
 error: Failed to parse value of `crate` as ident.
-  --> tests/fail/macros_invalid_input.rs:38:23
+  --> $DIR/macros_invalid_input.rs:38:23
    |
 38 | #[tokio::test(crate = 456)]
    |                       ^^^
 
 error: Failed to parse value of `crate` as ident: "456"
-  --> tests/fail/macros_invalid_input.rs:41:23
+  --> $DIR/macros_invalid_input.rs:41:23
    |
 41 | #[tokio::test(crate = "456")]
    |                       ^^^^^
 
 error: Failed to parse value of `crate` as ident: "abc::edf"
-  --> tests/fail/macros_invalid_input.rs:44:23
+  --> $DIR/macros_invalid_input.rs:44:23
    |
 44 | #[tokio::test(crate = "abc::edf")]
    |                       ^^^^^^^^^^
 
 error: second test attribute is supplied
-  --> tests/fail/macros_invalid_input.rs:48:1
+  --> $DIR/macros_invalid_input.rs:48:1
    |
 48 | #[test]
    | ^^^^^^^
 
 error: duplicated attribute
-  --> tests/fail/macros_invalid_input.rs:48:1
+  --> $DIR/macros_invalid_input.rs:48:1
    |
 48 | #[test]
    | ^^^^^^^
    |
 note: the lint level is defined here
-  --> tests/fail/macros_invalid_input.rs:1:9
+  --> $DIR/macros_invalid_input.rs:1:9
    |
 1  | #![deny(duplicate_macro_attributes)]
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^

tests-build/tests/fail/macros_type_mismatch.stderr

@@ -1,8 +1,8 @@
 error[E0308]: mismatched types
- --> tests/fail/macros_type_mismatch.rs:5:5
+ --> $DIR/macros_type_mismatch.rs:5:5
   |
 4 | async fn missing_semicolon_or_return_type() {
-  |                                             - possibly return type missing here?
+  |                                             - help: a return type might be missing here: `-> _`
 5 |     Ok(())
   |     ^^^^^^ expected `()`, found enum `Result`
   |
@@ -10,10 +10,10 @@ error[E0308]: mismatched types
                   found enum `Result<(), _>`
 
 error[E0308]: mismatched types
-  --> tests/fail/macros_type_mismatch.rs:10:5
+  --> $DIR/macros_type_mismatch.rs:10:5
    |
 9  | async fn missing_return_type() {
-   |                                - possibly return type missing here?
+   |                                - help: a return type might be missing here: `-> _`
 10 |     return Ok(());
    |     ^^^^^^^^^^^^^^ expected `()`, found enum `Result`
    |
@@ -21,7 +21,7 @@ error[E0308]: mismatched types
                    found enum `Result<(), _>`
 
 error[E0308]: mismatched types
-  --> tests/fail/macros_type_mismatch.rs:23:5
+  --> $DIR/macros_type_mismatch.rs:23:5
    |
 14 | async fn extra_semicolon() -> Result<(), ()> {
    |                               -------------- expected `Result<(), ()>` because of return type
@@ -31,18 +31,17 @@ error[E0308]: mismatched types
    |
    = note:   expected enum `Result<(), ()>`
            found unit type `()`
-help: try wrapping the expression in a variant of `Result`
+help: try adding an expression at the end of the block
+   |
+23 ~     Ok(());;
+24 +     Ok(())
    |
-23 |     Ok(Ok(());)
-   |     +++       +
-23 |     Err(Ok(());)
-   |     ++++       +
 
 error[E0308]: mismatched types
-  --> tests/fail/macros_type_mismatch.rs:32:5
+  --> $DIR/macros_type_mismatch.rs:32:5
    |
 30 | async fn issue_4635() {
-   |                       - possibly return type missing here?
+   |                       - help: try adding a return type: `-> i32`
 31 |     return 1;
 32 |     ;
    |     ^ expected `()`, found integer

tests-integration/Cargo.toml

@@ -16,11 +16,29 @@ required-features = ["rt-net"]
 name = "test-process-signal"
 required-features = ["rt-process-signal"]
 
+[[test]]
+name = "macros_main"
+
+[[test]]
+name = "macros_pin"
+
+[[test]]
+name = "macros_select"
+
+[[test]]
+name = "rt_yield"
+required-features = ["rt", "macros", "sync"]
+
 [features]
 # For mem check
 rt-net = ["tokio/rt", "tokio/rt-multi-thread", "tokio/net"]
 # For test-process-signal
 rt-process-signal = ["rt-net", "tokio/process", "tokio/signal"]
+# For testing wasi + rt/macros/sync features
+#
+# This is an explicit feature so we can use `cargo hack` testing single features
+# instead of all possible permutations.
+wasi-rt = ["rt", "macros", "sync"]
 
 full = [
     "macros",

tests-integration/tests/macros_main.rs

@@ -1,4 +1,8 @@
-#![cfg(all(feature = "macros", feature = "rt"))]
+#![cfg(all(
+    feature = "macros",
+    feature = "rt-multi-thread",
+    not(target_os = "wasi")
+))]
 
 #[tokio::main]
 async fn basic_main() -> usize {

tests-integration/tests/macros_select.rs

@@ -4,6 +4,7 @@ use futures::channel::oneshot;
 use futures::executor::block_on;
 use std::thread;
 
+#[cfg_attr(target_os = "wasi", ignore = "WASI: std::thread::spawn not supported")]
 #[test]
 fn join_with_select() {
     block_on(async {

tests-integration/tests/process_stdio.rs

@@ -1,5 +1,5 @@
 #![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(target_os = "wasi")))]
 
 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
 use tokio::join;

tests-integration/tests/rt_yield.rs

@@ -0,0 +1,41 @@
+use tokio::sync::oneshot;
+use tokio::task;
+
+async fn spawn_send() {
+    let (tx, rx) = oneshot::channel();
+
+    let task = tokio::spawn(async {
+        for _ in 0..10 {
+            task::yield_now().await;
+        }
+
+        tx.send("done").unwrap();
+    });
+
+    assert_eq!("done", rx.await.unwrap());
+    task.await.unwrap();
+}
+
+#[tokio::main(flavor = "current_thread")]
+async fn entry_point() {
+    spawn_send().await;
+}
+
+#[tokio::test]
+async fn test_macro() {
+    spawn_send().await;
+}
+
+#[test]
+fn main_macro() {
+    entry_point();
+}
+
+#[test]
+fn manual_rt() {
+    let rt = tokio::runtime::Builder::new_current_thread()
+        .build()
+        .unwrap();
+
+    rt.block_on(async { spawn_send().await });
+}

tokio-macros/CHANGELOG.md

@@ -1,6 +1,14 @@
+# 1.8.0 (June 4th, 2022)
+
+- macros: always emit return statement ([#4636])
+- macros: support setting a custom crate name for `#[tokio::main]` and `#[tokio::test]` ([#4613])
+
+[#4613]: https://github.com/tokio-rs/tokio/pull/4613
+[#4636]: https://github.com/tokio-rs/tokio/pull/4636
+
 # 1.7.0 (December 15th, 2021)
 
-- macros: address remainging clippy::semicolon_if_nothing_returned warning ([#4252])
+- macros: address remaining `clippy::semicolon_if_nothing_returned` warning ([#4252])
 
 [#4252]: https://github.com/tokio-rs/tokio/pull/4252
 

tokio-macros/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-macros"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-macros-1.0.x" git tag.
-version = "1.7.0"
+version = "1.8.0"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]

tokio-macros/src/lib.rs

@@ -265,11 +265,20 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
     entry::main(args, item, false)
 }
 
-/// Marks async function to be executed by runtime, suitable to test environment
+/// Marks async function to be executed by runtime, suitable to test environment.
+/// This macro helps set up a `Runtime` without requiring the user to use
+/// [Runtime](../tokio/runtime/struct.Runtime.html) or
+/// [Builder](../tokio/runtime/struct.Builder.html) directly.
 ///
-/// ## Usage
+/// Note: This macro is designed to be simplistic and targets applications that
+/// do not require a complex setup. If the provided functionality is not
+/// sufficient, you may be interested in using
+/// [Builder](../tokio/runtime/struct.Builder.html), which provides a more
+/// powerful interface.
 ///
-/// ### Multi-thread runtime
+/// # Multi-threaded runtime
+///
+/// To use the multi-threaded runtime, the macro can be configured using
 ///
 /// ```no_run
 /// #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -278,9 +287,17 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
 /// }
 /// ```
 ///
-/// ### Using default
+/// The `worker_threads` option configures the number of worker threads, and
+/// defaults to the number of cpus on the system. This is the default
+/// flavor.
 ///
-/// The default test runtime is single-threaded.
+/// Note: The multi-threaded runtime requires the `rt-multi-thread` feature
+/// flag.
+///
+/// # Current thread runtime
+///
+/// The default test runtime is single-threaded. Each test gets a
+/// separate current-thread runtime.
 ///
 /// ```no_run
 /// #[tokio::test]
@@ -289,6 +306,81 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
 /// }
 /// ```
 ///
+/// ## Usage
+///
+/// ### Using the multi-thread runtime
+///
+/// ```no_run
+/// #[tokio::test(flavor = "multi_thread")]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::test]`
+///
+/// ```no_run
+/// #[test]
+/// fn my_test() {
+///     tokio::runtime::Builder::new_multi_thread()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             assert!(true);
+///         })
+/// }
+/// ```
+///
+/// ### Using current thread runtime
+///
+/// ```no_run
+/// #[tokio::test]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::test]`
+///
+/// ```no_run
+/// #[test]
+/// fn my_test() {
+///     tokio::runtime::Builder::new_current_thread()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             assert!(true);
+///         })
+/// }
+/// ```
+///
+/// ### Set number of worker threads
+///
+/// ```no_run
+/// #[tokio::test(flavor ="multi_thread", worker_threads = 2)]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::test]`
+///
+/// ```no_run
+/// #[test]
+/// fn my_test() {
+///     tokio::runtime::Builder::new_multi_thread()
+///         .worker_threads(2)
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             assert!(true);
+///         })
+/// }
+/// ```
+///
 /// ### Configure the runtime to start with time paused
 ///
 /// ```no_run
@@ -298,6 +390,22 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
 /// }
 /// ```
 ///
+/// Equivalent code not using `#[tokio::test]`
+///
+/// ```no_run
+/// #[test]
+/// fn my_test() {
+///     tokio::runtime::Builder::new_current_thread()
+///         .enable_all()
+///         .start_paused(true)
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             assert!(true);
+///         })
+/// }
+/// ```
+///
 /// Note that `start_paused` requires the `test-util` feature to be enabled.
 ///
 /// ### Rename package

tokio-stream/CHANGELOG.md

@@ -1,3 +1,23 @@
+# 0.1.9 (June 4, 2022)
+
+- deps: upgrade `tokio-util` dependency to `0.7.x` ([#3762])
+- stream: add `StreamExt::map_while` ([#4351])
+- stream: add `StreamExt::then` ([#4355])
+- stream: add cancel-safety docs to `StreamExt::next` and `try_next` ([#4715])
+- stream: expose `Elapsed` error ([#4502])
+- stream: expose `Timeout` ([#4601])
+- stream: implement `Extend` for `StreamMap` ([#4272])
+- sync: add `Clone` to `RecvError` types ([#4560])
+
+[#3762]: https://github.com/tokio-rs/tokio/pull/3762
+[#4272]: https://github.com/tokio-rs/tokio/pull/4272
+[#4351]: https://github.com/tokio-rs/tokio/pull/4351
+[#4355]: https://github.com/tokio-rs/tokio/pull/4355
+[#4502]: https://github.com/tokio-rs/tokio/pull/4502
+[#4560]: https://github.com/tokio-rs/tokio/pull/4560
+[#4601]: https://github.com/tokio-rs/tokio/pull/4601
+[#4715]: https://github.com/tokio-rs/tokio/pull/4715
+
 # 0.1.8 (October 29, 2021)
 
 - stream: add `From<Receiver<T>>` impl for receiver streams ([#4080])

tokio-stream/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-stream"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-stream-0.1.x" git tag.
-version = "0.1.8"
+version = "0.1.9"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -34,9 +34,11 @@ tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true }
 [dev-dependencies]
 tokio = { version = "1.2.0", path = "../tokio", features = ["full", "test-util"] }
 async-stream = "0.3"
+parking_lot = "0.12.0"
 tokio-test = { path = "../tokio-test" }
 futures = { version = "0.3", default-features = false }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 proptest = "1"
 
 [package.metadata.docs.rs]

tokio-stream/src/stream_ext.rs

@@ -61,6 +61,8 @@ cfg_time! {
     use tokio::time::Duration;
     mod throttle;
     use throttle::{throttle, Throttle};
+    mod chunks_timeout;
+    use chunks_timeout::ChunksTimeout;
 }
 
 /// An extension trait for the [`Stream`] trait that provides a variety of
@@ -113,6 +115,12 @@ pub trait StreamExt: Stream {
     /// pinning it to the stack using the `pin_mut!` macro from the `pin_utils`
     /// crate.
     ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. The returned future only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     /// # Examples
     ///
     /// ```
@@ -149,6 +157,12 @@ pub trait StreamExt: Stream {
     /// an [`Option<Result<T, E>>`](Option), making for easy use
     /// with the [`?`](std::ops::Try) operator.
     ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. The returned future only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     /// # Examples
     ///
     /// ```
@@ -993,6 +1007,63 @@ pub trait StreamExt: Stream {
     {
         throttle(duration, self)
     }
+
+    /// Batches the items in the given stream using a maximum duration and size for each batch.
+    ///
+    /// This stream returns the next batch of items in the following situations:
+    ///  1. The inner stream has returned at least `max_size` many items since the last batch.
+    ///  2. The time since the first item of a batch is greater than the given duration.
+    ///  3. The end of the stream is reached.
+    ///
+    /// The length of the returned vector is never empty or greater than the maximum size. Empty batches
+    /// will not be emitted if no items are received upstream.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if `max_size` is zero
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// use std::time::Duration;
+    /// use tokio::time;
+    /// use tokio_stream::{self as stream, StreamExt};
+    /// use futures::FutureExt;
+    ///
+    /// #[tokio::main]
+    /// # async fn _unused() {}
+    /// # #[tokio::main(flavor = "current_thread", start_paused = true)]
+    /// async fn main() {
+    ///     let iter = vec![1, 2, 3, 4].into_iter();
+    ///     let stream0 = stream::iter(iter);
+    ///
+    ///     let iter = vec![5].into_iter();
+    ///     let stream1 = stream::iter(iter)
+    ///          .then(move |n| time::sleep(Duration::from_secs(5)).map(move |_| n));
+    ///
+    ///     let chunk_stream = stream0
+    ///         .chain(stream1)
+    ///         .chunks_timeout(3, Duration::from_secs(2));
+    ///     tokio::pin!(chunk_stream);
+    ///
+    ///     // a full batch was received
+    ///     assert_eq!(chunk_stream.next().await, Some(vec![1,2,3]));
+    ///     // deadline was reached before max_size was reached
+    ///     assert_eq!(chunk_stream.next().await, Some(vec![4]));
+    ///     // last element in the stream
+    ///     assert_eq!(chunk_stream.next().await, Some(vec![5]));
+    /// }
+    /// ```
+    #[cfg(feature = "time")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
+    #[track_caller]
+    fn chunks_timeout(self, max_size: usize, duration: Duration) -> ChunksTimeout<Self>
+    where
+        Self: Sized,
+    {
+        assert!(max_size > 0, "`max_size` must be non-zero.");
+        ChunksTimeout::new(self, max_size, duration)
+    }
 }
 
 impl<St: ?Sized> StreamExt for St where St: Stream {}
@@ -1000,10 +1071,10 @@ impl<St: ?Sized> StreamExt for St where St: Stream {}
 /// Merge the size hints from two streams.
 fn merge_size_hints(
     (left_low, left_high): (usize, Option<usize>),
-    (right_low, right_hign): (usize, Option<usize>),
+    (right_low, right_high): (usize, Option<usize>),
 ) -> (usize, Option<usize>) {
     let low = left_low.saturating_add(right_low);
-    let high = match (left_high, right_hign) {
+    let high = match (left_high, right_high) {
         (Some(h1), Some(h2)) => h1.checked_add(h2),
         _ => None,
     };

tokio-stream/src/stream_ext/chunks_timeout.rs

@@ -0,0 +1,84 @@
+use crate::stream_ext::Fuse;
+use crate::Stream;
+use tokio::time::{sleep, Instant, Sleep};
+
+use core::future::Future;
+use core::pin::Pin;
+use core::task::{Context, Poll};
+use pin_project_lite::pin_project;
+use std::time::Duration;
+
+pin_project! {
+    /// Stream returned by the [`chunks_timeout`](super::StreamExt::chunks_timeout) method.
+    #[must_use = "streams do nothing unless polled"]
+    #[derive(Debug)]
+    pub struct ChunksTimeout<S: Stream> {
+        #[pin]
+        stream: Fuse<S>,
+        #[pin]
+        deadline: Sleep,
+        duration: Duration,
+        items: Vec<S::Item>,
+        cap: usize, // https://github.com/rust-lang/futures-rs/issues/1475
+    }
+}
+
+impl<S: Stream> ChunksTimeout<S> {
+    pub(super) fn new(stream: S, max_size: usize, duration: Duration) -> Self {
+        ChunksTimeout {
+            stream: Fuse::new(stream),
+            deadline: sleep(duration),
+            duration,
+            items: Vec::with_capacity(max_size),
+            cap: max_size,
+        }
+    }
+}
+
+impl<S: Stream> Stream for ChunksTimeout<S> {
+    type Item = Vec<S::Item>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let mut me = self.as_mut().project();
+        loop {
+            match me.stream.as_mut().poll_next(cx) {
+                Poll::Pending => break,
+                Poll::Ready(Some(item)) => {
+                    if me.items.is_empty() {
+                        me.deadline.as_mut().reset(Instant::now() + *me.duration);
+                        me.items.reserve_exact(*me.cap);
+                    }
+                    me.items.push(item);
+                    if me.items.len() >= *me.cap {
+                        return Poll::Ready(Some(std::mem::take(me.items)));
+                    }
+                }
+                Poll::Ready(None) => {
+                    // Returning Some here is only correct because we fuse the inner stream.
+                    let last = if me.items.is_empty() {
+                        None
+                    } else {
+                        Some(std::mem::take(me.items))
+                    };
+
+                    return Poll::Ready(last);
+                }
+            }
+        }
+
+        if !me.items.is_empty() {
+            ready!(me.deadline.poll(cx));
+            return Poll::Ready(Some(std::mem::take(me.items)));
+        }
+
+        Poll::Pending
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let chunk_len = if self.items.is_empty() { 0 } else { 1 };
+        let (lower, upper) = self.stream.size_hint();
+        let lower = (lower / self.cap).saturating_add(chunk_len);
+        let upper = upper.and_then(|x| x.checked_add(chunk_len));
+        (lower, upper)
+    }
+}

tokio-stream/src/stream_ext/next.rs

@@ -8,6 +8,13 @@ use pin_project_lite::pin_project;
 
 pin_project! {
     /// Future for the [`next`](super::StreamExt::next) method.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. It only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     #[derive(Debug)]
     #[must_use = "futures do nothing unless you `.await` or poll them"]
     pub struct Next<'a, St: ?Sized> {

tokio-stream/src/stream_ext/try_next.rs

@@ -9,6 +9,12 @@ use pin_project_lite::pin_project;
 
 pin_project! {
     /// Future for the [`try_next`](super::StreamExt::try_next) method.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. It only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
     #[derive(Debug)]
     #[must_use = "futures do nothing unless you `.await` or poll them"]
     pub struct TryNext<'a, St: ?Sized> {

tokio-stream/tests/chunks_timeout.rs

@@ -0,0 +1,84 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "time", feature = "sync", feature = "io-util"))]
+
+use tokio::time;
+use tokio_stream::{self as stream, StreamExt};
+use tokio_test::assert_pending;
+use tokio_test::task;
+
+use futures::FutureExt;
+use std::time::Duration;
+
+#[tokio::test(start_paused = true)]
+async fn usage() {
+    let iter = vec![1, 2, 3].into_iter();
+    let stream0 = stream::iter(iter);
+
+    let iter = vec![4].into_iter();
+    let stream1 =
+        stream::iter(iter).then(move |n| time::sleep(Duration::from_secs(3)).map(move |_| n));
+
+    let chunk_stream = stream0
+        .chain(stream1)
+        .chunks_timeout(4, Duration::from_secs(2));
+
+    let mut chunk_stream = task::spawn(chunk_stream);
+
+    assert_pending!(chunk_stream.poll_next());
+    time::advance(Duration::from_secs(2)).await;
+    assert_eq!(chunk_stream.next().await, Some(vec![1, 2, 3]));
+
+    assert_pending!(chunk_stream.poll_next());
+    time::advance(Duration::from_secs(2)).await;
+    assert_eq!(chunk_stream.next().await, Some(vec![4]));
+}
+
+#[tokio::test(start_paused = true)]
+async fn full_chunk_with_timeout() {
+    let iter = vec![1, 2].into_iter();
+    let stream0 = stream::iter(iter);
+
+    let iter = vec![3].into_iter();
+    let stream1 =
+        stream::iter(iter).then(move |n| time::sleep(Duration::from_secs(1)).map(move |_| n));
+
+    let iter = vec![4].into_iter();
+    let stream2 =
+        stream::iter(iter).then(move |n| time::sleep(Duration::from_secs(3)).map(move |_| n));
+
+    let chunk_stream = stream0
+        .chain(stream1)
+        .chain(stream2)
+        .chunks_timeout(3, Duration::from_secs(2));
+
+    let mut chunk_stream = task::spawn(chunk_stream);
+
+    assert_pending!(chunk_stream.poll_next());
+    time::advance(Duration::from_secs(2)).await;
+    assert_eq!(chunk_stream.next().await, Some(vec![1, 2, 3]));
+
+    assert_pending!(chunk_stream.poll_next());
+    time::advance(Duration::from_secs(2)).await;
+    assert_eq!(chunk_stream.next().await, Some(vec![4]));
+}
+
+#[tokio::test]
+#[ignore]
+async fn real_time() {
+    let iter = vec![1, 2, 3, 4].into_iter();
+    let stream0 = stream::iter(iter);
+
+    let iter = vec![5].into_iter();
+    let stream1 =
+        stream::iter(iter).then(move |n| time::sleep(Duration::from_secs(5)).map(move |_| n));
+
+    let chunk_stream = stream0
+        .chain(stream1)
+        .chunks_timeout(3, Duration::from_secs(2));
+
+    let mut chunk_stream = task::spawn(chunk_stream);
+
+    assert_eq!(chunk_stream.next().await, Some(vec![1, 2, 3]));
+    assert_eq!(chunk_stream.next().await, Some(vec![4]));
+    assert_eq!(chunk_stream.next().await, Some(vec![5]));
+}

tokio-stream/tests/stream_panic.rs

@@ -0,0 +1,55 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "time", not(target_os = "wasi")))] // Wasi does not support panic recovery
+
+use parking_lot::{const_mutex, Mutex};
+use std::error::Error;
+use std::panic;
+use std::sync::Arc;
+use tokio::time::Duration;
+use tokio_stream::{self as stream, StreamExt};
+
+fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<String> {
+    static PANIC_MUTEX: Mutex<()> = const_mutex(());
+
+    {
+        let _guard = PANIC_MUTEX.lock();
+        let panic_file: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
+
+        let prev_hook = panic::take_hook();
+        {
+            let panic_file = panic_file.clone();
+            panic::set_hook(Box::new(move |panic_info| {
+                let panic_location = panic_info.location().unwrap();
+                panic_file
+                    .lock()
+                    .clone_from(&Some(panic_location.file().to_string()));
+            }));
+        }
+
+        let result = panic::catch_unwind(func);
+        // Return to the previously set panic hook (maybe default) so that we get nice error
+        // messages in the tests.
+        panic::set_hook(prev_hook);
+
+        if result.is_err() {
+            panic_file.lock().clone()
+        } else {
+            None
+        }
+    }
+}
+
+#[test]
+fn stream_chunks_timeout_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let iter = vec![1, 2, 3].into_iter();
+        let stream0 = stream::iter(iter);
+
+        let _chunk_stream = stream0.chunks_timeout(0, Duration::from_secs(2));
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}

tokio-stream/tests/stream_stream_map.rs

@@ -325,6 +325,7 @@ fn one_ready_many_none() {
     }
 }
 
+#[cfg(not(target_os = "wasi"))]
 proptest::proptest! {
     #[test]
     fn fuzz_pending_complete_mix(kinds: Vec<bool>) {

tokio-test/src/macros.rs

@@ -260,7 +260,7 @@ macro_rules! assert_err {
     }};
 }
 
-/// Asserts that an exact duration has elapsed since since the start instant ±1ms.
+/// Asserts that an exact duration has elapsed since the start instant ±1ms.
 ///
 /// ```rust
 /// use tokio::time::{self, Instant};

tokio-util/CHANGELOG.md

@@ -1,3 +1,47 @@
+# 0.7.4 (September 8, 2022)
+
+### Added
+
+- io: add `SyncIoBridge::shutdown()` ([#4938])
+- task: improve `LocalPoolHandle` ([#4680])
+
+### Fixed
+
+- util: add `track_caller` to public APIs ([#4785])
+
+### Unstable
+
+- task: fix compilation errors in `JoinMap` with Tokio v1.21.0 ([#4755])
+- task: remove the unstable, deprecated `JoinMap::join_one` ([#4920])
+
+[#4680]: https://github.com/tokio-rs/tokio/pull/4680
+[#4755]: https://github.com/tokio-rs/tokio/pull/4755
+[#4785]: https://github.com/tokio-rs/tokio/pull/4785
+[#4920]: https://github.com/tokio-rs/tokio/pull/4920
+[#4938]: https://github.com/tokio-rs/tokio/pull/4938
+
+# 0.7.3 (June 4, 2022)
+
+### Changed
+
+- tracing: don't require default tracing features ([#4592])
+- util: simplify implementation of `ReusableBoxFuture` ([#4675])
+
+### Added (unstable)
+
+- task: add `JoinMap` ([#4640], [#4697])
+
+[#4592]: https://github.com/tokio-rs/tokio/pull/4592
+[#4640]: https://github.com/tokio-rs/tokio/pull/4640
+[#4675]: https://github.com/tokio-rs/tokio/pull/4675
+[#4697]: https://github.com/tokio-rs/tokio/pull/4697
+
+# 0.7.2 (May 14, 2022)
+
+This release contains a rewrite of `CancellationToken` that fixes a memory leak. ([#4652])
+
+[#4652]: https://github.com/tokio-rs/tokio/pull/4652
+
 # 0.7.1 (February 21, 2022)
 
 ### Added
@@ -49,6 +93,12 @@
 [#4214]: https://github.com/tokio-rs/tokio/pull/4214
 [#4241]: https://github.com/tokio-rs/tokio/pull/4241
 
+# 0.6.10 (May 14, 2021)
+
+This is a backport for the memory leak in `CancellationToken` that was originally fixed in 0.7.2. ([#4652])
+
+[#4652]: https://github.com/tokio-rs/tokio/pull/4652
+
 # 0.6.9 (October 29, 2021)
 
 ### Added

tokio-util/Cargo.toml

@@ -4,7 +4,7 @@ name = "tokio-util"
 # - Remove path dependencies
 # - Update CHANGELOG.md.
 # - Create "tokio-util-0.7.x" git tag.
-version = "0.7.1"
+version = "0.7.4"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -34,7 +34,7 @@ rt = ["tokio/rt", "tokio/sync", "futures-util", "hashbrown"]
 __docs_rs = ["futures-util"]
 
 [dependencies]
-tokio = { version = "1.18.0", path = "../tokio", features = ["sync"] }
+tokio = { version = "1.21.0", path = "../tokio", features = ["sync"] }
 bytes = "1.0.0"
 futures-core = "0.3.0"
 futures-sink = "0.3.0"
@@ -55,7 +55,12 @@ tokio-stream = { version = "0.1", path = "../tokio-stream" }
 async-stream = "0.3.0"
 futures = "0.3.0"
 futures-test = "0.3.5"
+parking_lot = "0.12.0"
 
 [package.metadata.docs.rs]
 all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
+# enable unstable features in the documentation
+rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]
+# it's necessary to _also_ pass `--cfg tokio_unstable` to rustc, or else
+# dependencies will not be enabled, and the docs build will fail.
+rustc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]

tokio-util/src/codec/decoder.rs

@@ -20,7 +20,7 @@ use std::io;
 /// it's possible to temporarily read 0 bytes by reaching EOF.
 ///
 /// In these cases `decode_eof` will be called until it signals
-/// fullfillment of all closing frames by returning `Ok(None)`.
+/// fulfillment of all closing frames by returning `Ok(None)`.
 /// After that, repeated attempts to read from the [`Framed`] or [`FramedRead`]
 /// will not invoke `decode` or `decode_eof` again, until data can be read
 /// during a retry.

tokio-util/src/io/stream_reader.rs

@@ -50,9 +50,58 @@ pin_project! {
     /// # }
     /// ```
     ///
+    /// If the stream produces errors which are not [std::io::Error],
+    /// the errors can be converted using [`StreamExt`] to map each
+    /// element.
+    ///
+    /// ```
+    /// use bytes::Bytes;
+    /// use tokio::io::AsyncReadExt;
+    /// use tokio_util::io::StreamReader;
+    /// use tokio_stream::StreamExt;
+    /// # #[tokio::main]
+    /// # async fn main() -> std::io::Result<()> {
+    ///
+    /// // Create a stream from an iterator, including an error.
+    /// let stream = tokio_stream::iter(vec![
+    ///     Result::Ok(Bytes::from_static(&[0, 1, 2, 3])),
+    ///     Result::Ok(Bytes::from_static(&[4, 5, 6, 7])),
+    ///     Result::Err("Something bad happened!")
+    /// ]);
+    ///
+    /// // Use StreamExt to map the stream and error to a std::io::Error
+    /// let stream = stream.map(|result| result.map_err(|err| {
+    ///     std::io::Error::new(std::io::ErrorKind::Other, err)
+    /// }));
+    ///
+    /// // Convert it to an AsyncRead.
+    /// let mut read = StreamReader::new(stream);
+    ///
+    /// // Read five bytes from the stream.
+    /// let mut buf = [0; 5];
+    /// read.read_exact(&mut buf).await?;
+    /// assert_eq!(buf, [0, 1, 2, 3, 4]);
+    ///
+    /// // Read the rest of the current chunk.
+    /// assert_eq!(read.read(&mut buf).await?, 3);
+    /// assert_eq!(&buf[..3], [5, 6, 7]);
+    ///
+    /// // Reading the next chunk will produce an error
+    /// let error = read.read(&mut buf).await.unwrap_err();
+    /// assert_eq!(error.kind(), std::io::ErrorKind::Other);
+    /// assert_eq!(error.into_inner().unwrap().to_string(), "Something bad happened!");
+    ///
+    /// // We have now reached the end.
+    /// assert_eq!(read.read(&mut buf).await?, 0);
+    ///
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
     /// [`AsyncRead`]: tokio::io::AsyncRead
     /// [`Stream`]: futures_core::Stream
     /// [`ReaderStream`]: crate::io::ReaderStream
+    /// [`StreamExt`]: tokio_stream::StreamExt
     #[derive(Debug)]
     pub struct StreamReader<S, B> {
         #[pin]
@@ -93,7 +142,7 @@ where
     }
 
     /// Consumes this `StreamReader`, returning a Tuple consisting
-    /// of the underlying stream and an Option of the interal buffer,
+    /// of the underlying stream and an Option of the internal buffer,
     /// which is Some in case the buffer contains elements.
     pub fn into_inner_with_chunk(self) -> (S, Option<B>) {
         if self.has_chunk() {

tokio-util/src/io/sync_bridge.rs

@@ -66,6 +66,21 @@ impl<T: AsyncWrite> SyncIoBridge<T> {
     }
 }
 
+impl<T: AsyncWrite + Unpin> SyncIoBridge<T> {
+    /// Shutdown this writer. This method provides a way to call the [`AsyncWriteExt::shutdown`]
+    /// function of the inner [`tokio::io::AsyncWrite`] instance.
+    ///
+    /// # Errors
+    ///
+    /// This method returns the same errors as [`AsyncWriteExt::shutdown`].
+    ///
+    /// [`AsyncWriteExt::shutdown`]: tokio::io::AsyncWriteExt::shutdown
+    pub fn shutdown(&mut self) -> std::io::Result<()> {
+        let src = &mut self.src;
+        self.rt.block_on(src.shutdown())
+    }
+}
+
 impl<T: Unpin> SyncIoBridge<T> {
     /// Use a [`tokio::io::AsyncRead`] synchronously as a [`std::io::Read`] or
     /// a [`tokio::io::AsyncWrite`] as a [`std::io::Write`].
@@ -85,9 +100,10 @@ impl<T: Unpin> SyncIoBridge<T> {
     ///
     /// Use e.g. `SyncIoBridge::new(Box::pin(src))`.
     ///
-    /// # Panic
+    /// # Panics
     ///
     /// This will panic if called outside the context of a Tokio runtime.
+    #[track_caller]
     pub fn new(src: T) -> Self {
         Self::new_with_handle(src, tokio::runtime::Handle::current())
     }

tokio-util/src/lib.rs

@@ -29,6 +29,7 @@ cfg_codec! {
 }
 
 cfg_net! {
+    #[cfg(not(target_arch = "wasm32"))]
     pub mod udp;
     pub mod net;
 }
@@ -115,6 +116,9 @@ mod util {
 
         let n = {
             let dst = buf.chunk_mut();
+
+            // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
+            // transparent wrapper around `[MaybeUninit<u8>]`.
             let dst = unsafe { &mut *(dst as *mut _ as *mut [MaybeUninit<u8>]) };
             let mut buf = ReadBuf::uninit(dst);
             let ptr = buf.filled().as_ptr();

tokio-util/src/sync/cancellation_token.rs

@@ -1,18 +1,15 @@
 //! An asynchronously awaitable `CancellationToken`.
 //! The token allows to signal a cancellation request to one or more tasks.
 pub(crate) mod guard;
+mod tree_node;
 
-use crate::loom::sync::atomic::AtomicUsize;
-use crate::loom::sync::Mutex;
-use crate::sync::intrusive_double_linked_list::{LinkedList, ListNode};
-
+use crate::loom::sync::Arc;
 use core::future::Future;
 use core::pin::Pin;
-use core::ptr::NonNull;
-use core::sync::atomic::Ordering;
-use core::task::{Context, Poll, Waker};
+use core::task::{Context, Poll};
 
 use guard::DropGuard;
+use pin_project_lite::pin_project;
 
 /// A token which can be used to signal a cancellation request to one or more
 /// tasks.
@@ -55,31 +52,20 @@ use guard::DropGuard;
 /// }
 /// ```
 pub struct CancellationToken {
-    inner: NonNull<CancellationTokenState>,
+    inner: Arc<tree_node::TreeNode>,
 }
 
-// Safety: The CancellationToken is thread-safe and can be moved between threads,
-// since all methods are internally synchronized.
-unsafe impl Send for CancellationToken {}
-unsafe impl Sync for CancellationToken {}
-
-/// A Future that is resolved once the corresponding [`CancellationToken`]
-/// was cancelled
-#[must_use = "futures do nothing unless polled"]
-pub struct WaitForCancellationFuture<'a> {
-    /// The CancellationToken that is associated with this WaitForCancellationFuture
-    cancellation_token: Option<&'a CancellationToken>,
-    /// Node for waiting at the cancellation_token
-    wait_node: ListNode<WaitQueueEntry>,
-    /// Whether this future was registered at the token yet as a waiter
-    is_registered: bool,
+pin_project! {
+    /// A Future that is resolved once the corresponding [`CancellationToken`]
+    /// is cancelled.
+    #[must_use = "futures do nothing unless polled"]
+    pub struct WaitForCancellationFuture<'a> {
+        cancellation_token: &'a CancellationToken,
+        #[pin]
+        future: tokio::sync::futures::Notified<'a>,
+    }
 }
 
-// Safety: Futures can be sent between threads as long as the underlying
-// cancellation_token is thread-safe (Sync),
-// which allows to poll/register/unregister from a different thread.
-unsafe impl<'a> Send for WaitForCancellationFuture<'a> {}
-
 // ===== impl CancellationToken =====
 
 impl core::fmt::Debug for CancellationToken {
@@ -92,43 +78,16 @@ impl core::fmt::Debug for CancellationToken {
 
 impl Clone for CancellationToken {
     fn clone(&self) -> Self {
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        let inner = self.state();
-
-        // Tokens are cloned by increasing their refcount
-        let current_state = inner.snapshot();
-        inner.increment_refcount(current_state);
-
-        CancellationToken { inner: self.inner }
+        tree_node::increase_handle_refcount(&self.inner);
+        CancellationToken {
+            inner: self.inner.clone(),
+        }
     }
 }
 
 impl Drop for CancellationToken {
     fn drop(&mut self) {
-        let token_state_pointer = self.inner;
-
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        let inner = unsafe { &mut *self.inner.as_ptr() };
-
-        let mut current_state = inner.snapshot();
-
-        // We need to safe the parent, since the state might be released by the
-        // next call
-        let parent = inner.parent;
-
-        // Drop our own refcount
-        current_state = inner.decrement_refcount(current_state);
-
-        // If this was the last reference, unregister from the parent
-        if current_state.refcount == 0 {
-            if let Some(mut parent) = parent {
-                // Safety: Since we still retain a reference on the parent, it must be valid.
-                let parent = unsafe { parent.as_mut() };
-                parent.unregister_child(token_state_pointer, current_state);
-            }
-        }
+        tree_node::decrease_handle_refcount(&self.inner);
     }
 }
 
@@ -141,29 +100,11 @@ impl Default for CancellationToken {
 impl CancellationToken {
     /// Creates a new CancellationToken in the non-cancelled state.
     pub fn new() -> CancellationToken {
-        let state = Box::new(CancellationTokenState::new(
-            None,
-            StateSnapshot {
-                cancel_state: CancellationState::NotCancelled,
-                has_parent_ref: false,
-                refcount: 1,
-            },
-        ));
-
-        // Safety: We just created the Box. The pointer is guaranteed to be
-        // not null
         CancellationToken {
-            inner: unsafe { NonNull::new_unchecked(Box::into_raw(state)) },
+            inner: Arc::new(tree_node::TreeNode::new()),
         }
     }
 
-    /// Returns a reference to the utilized `CancellationTokenState`.
-    fn state(&self) -> &CancellationTokenState {
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        unsafe { &*self.inner.as_ptr() }
-    }
-
     /// Creates a `CancellationToken` which will get cancelled whenever the
     /// current token gets cancelled.
     ///
@@ -203,56 +144,8 @@ impl CancellationToken {
     /// }
     /// ```
     pub fn child_token(&self) -> CancellationToken {
-        let inner = self.state();
-
-        // Increment the refcount of this token. It will be referenced by the
-        // child, independent of whether the child is immediately cancelled or
-        // not.
-        let _current_state = inner.increment_refcount(inner.snapshot());
-
-        let mut unpacked_child_state = StateSnapshot {
-            has_parent_ref: true,
-            refcount: 1,
-            cancel_state: CancellationState::NotCancelled,
-        };
-        let mut child_token_state = Box::new(CancellationTokenState::new(
-            Some(self.inner),
-            unpacked_child_state,
-        ));
-
-        {
-            let mut guard = inner.synchronized.lock().unwrap();
-            if guard.is_cancelled {
-                // This task was already cancelled. In this case we should not
-                // insert the child into the list, since it would never get removed
-                // from the list.
-                (*child_token_state.synchronized.lock().unwrap()).is_cancelled = true;
-                unpacked_child_state.cancel_state = CancellationState::Cancelled;
-                // Since it's not in the list, the parent doesn't need to retain
-                // a reference to it.
-                unpacked_child_state.has_parent_ref = false;
-                child_token_state
-                    .state
-                    .store(unpacked_child_state.pack(), Ordering::SeqCst);
-            } else {
-                if let Some(mut first_child) = guard.first_child {
-                    child_token_state.from_parent.next_peer = Some(first_child);
-                    // Safety: We manipulate other child task inside the Mutex
-                    // and retain a parent reference on it. The child token can't
-                    // get invalidated while the Mutex is held.
-                    unsafe {
-                        first_child.as_mut().from_parent.prev_peer =
-                            Some((&mut *child_token_state).into())
-                    };
-                }
-                guard.first_child = Some((&mut *child_token_state).into());
-            }
-        };
-
-        let child_token_ptr = Box::into_raw(child_token_state);
-        // Safety: We just created the pointer from a `Box`
         CancellationToken {
-            inner: unsafe { NonNull::new_unchecked(child_token_ptr) },
+            inner: tree_node::child_node(&self.inner),
         }
     }
 
@@ -260,21 +153,33 @@ impl CancellationToken {
     /// derived from it.
     ///
     /// This will wake up all tasks which are waiting for cancellation.
+    ///
+    /// Be aware that cancellation is not an atomic operation. It is possible
+    /// for another thread running in parallel with a call to `cancel` to first
+    /// receive `true` from `is_cancelled` on one child node, and then receive
+    /// `false` from `is_cancelled` on another child node. However, once the
+    /// call to `cancel` returns, all child nodes have been fully cancelled.
     pub fn cancel(&self) {
-        self.state().cancel();
+        tree_node::cancel(&self.inner);
     }
 
-    /// Returns `true` if the `CancellationToken` had been cancelled
+    /// Returns `true` if the `CancellationToken` is cancelled.
     pub fn is_cancelled(&self) -> bool {
-        self.state().is_cancelled()
+        tree_node::is_cancelled(&self.inner)
     }
 
     /// Returns a `Future` that gets fulfilled when cancellation is requested.
+    ///
+    /// The future will complete immediately if the token is already cancelled
+    /// when this method is called.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe.
     pub fn cancelled(&self) -> WaitForCancellationFuture<'_> {
         WaitForCancellationFuture {
-            cancellation_token: Some(self),
-            wait_node: ListNode::new(WaitQueueEntry::new()),
-            is_registered: false,
+            cancellation_token: self,
+            future: self.inner.notified(),
         }
     }
 
@@ -285,26 +190,6 @@ impl CancellationToken {
     pub fn drop_guard(self) -> DropGuard {
         DropGuard { inner: Some(self) }
     }
-
-    unsafe fn register(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        self.state().register(wait_node, cx)
-    }
-
-    fn check_for_cancellation(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        self.state().check_for_cancellation(wait_node, cx)
-    }
-
-    fn unregister(&self, wait_node: &mut ListNode<WaitQueueEntry>) {
-        self.state().unregister(wait_node)
-    }
 }
 
 // ===== impl WaitForCancellationFuture =====
@@ -319,560 +204,21 @@ impl<'a> Future for WaitForCancellationFuture<'a> {
     type Output = ();
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-        // Safety: We do not move anything out of `WaitForCancellationFuture`
-        let mut_self: &mut WaitForCancellationFuture<'_> = unsafe { Pin::get_unchecked_mut(self) };
-
-        let cancellation_token = mut_self
-            .cancellation_token
-            .expect("polled WaitForCancellationFuture after completion");
-
-        let poll_res = if !mut_self.is_registered {
-            // Safety: The `ListNode` is pinned through the Future,
-            // and we will unregister it in `WaitForCancellationFuture::drop`
-            // before the Future is dropped and the memory reference is invalidated.
-            unsafe { cancellation_token.register(&mut mut_self.wait_node, cx) }
-        } else {
-            cancellation_token.check_for_cancellation(&mut mut_self.wait_node, cx)
-        };
-
-        if let Poll::Ready(()) = poll_res {
-            // The cancellation_token was signalled
-            mut_self.cancellation_token = None;
-            // A signalled Token means the Waker won't be enqueued anymore
-            mut_self.is_registered = false;
-            mut_self.wait_node.task = None;
-        } else {
-            // This `Future` and its stored `Waker` stay registered at the
-            // `CancellationToken`
-            mut_self.is_registered = true;
-        }
-
-        poll_res
-    }
-}
-
-impl<'a> Drop for WaitForCancellationFuture<'a> {
-    fn drop(&mut self) {
-        // If this WaitForCancellationFuture has been polled and it was added to the
-        // wait queue at the cancellation_token, it must be removed before dropping.
-        // Otherwise the cancellation_token would access invalid memory.
-        if let Some(token) = self.cancellation_token {
-            if self.is_registered {
-                token.unregister(&mut self.wait_node);
-            }
-        }
-    }
-}
-
-/// Tracks how the future had interacted with the [`CancellationToken`]
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum PollState {
-    /// The task has never interacted with the [`CancellationToken`].
-    New,
-    /// The task was added to the wait queue at the [`CancellationToken`].
-    Waiting,
-    /// The task has been polled to completion.
-    Done,
-}
-
-/// Tracks the WaitForCancellationFuture waiting state.
-/// Access to this struct is synchronized through the mutex in the CancellationToken.
-struct WaitQueueEntry {
-    /// The task handle of the waiting task
-    task: Option<Waker>,
-    // Current polling state. This state is only updated inside the Mutex of
-    // the CancellationToken.
-    state: PollState,
-}
-
-impl WaitQueueEntry {
-    /// Creates a new WaitQueueEntry
-    fn new() -> WaitQueueEntry {
-        WaitQueueEntry {
-            task: None,
-            state: PollState::New,
-        }
-    }
-}
-
-struct SynchronizedState {
-    waiters: LinkedList<WaitQueueEntry>,
-    first_child: Option<NonNull<CancellationTokenState>>,
-    is_cancelled: bool,
-}
-
-impl SynchronizedState {
-    fn new() -> Self {
-        Self {
-            waiters: LinkedList::new(),
-            first_child: None,
-            is_cancelled: false,
-        }
-    }
-}
-
-/// Information embedded in child tokens which is synchronized through the Mutex
-/// in their parent.
-struct SynchronizedThroughParent {
-    next_peer: Option<NonNull<CancellationTokenState>>,
-    prev_peer: Option<NonNull<CancellationTokenState>>,
-}
-
-/// Possible states of a `CancellationToken`
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum CancellationState {
-    NotCancelled = 0,
-    Cancelling = 1,
-    Cancelled = 2,
-}
-
-impl CancellationState {
-    fn pack(self) -> usize {
-        self as usize
-    }
-
-    fn unpack(value: usize) -> Self {
-        match value {
-            0 => CancellationState::NotCancelled,
-            1 => CancellationState::Cancelling,
-            2 => CancellationState::Cancelled,
-            _ => unreachable!("Invalid value"),
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-struct StateSnapshot {
-    /// The amount of references to this particular CancellationToken.
-    /// `CancellationToken` structs hold these references to a `CancellationTokenState`.
-    /// Also the state is referenced by the state of each child.
-    refcount: usize,
-    /// Whether the state is still referenced by it's parent and can therefore
-    /// not be freed.
-    has_parent_ref: bool,
-    /// Whether the token is cancelled
-    cancel_state: CancellationState,
-}
-
-impl StateSnapshot {
-    /// Packs the snapshot into a `usize`
-    fn pack(self) -> usize {
-        self.refcount << 3 | if self.has_parent_ref { 4 } else { 0 } | self.cancel_state.pack()
-    }
-
-    /// Unpacks the snapshot from a `usize`
-    fn unpack(value: usize) -> Self {
-        let refcount = value >> 3;
-        let has_parent_ref = value & 4 != 0;
-        let cancel_state = CancellationState::unpack(value & 0x03);
-
-        StateSnapshot {
-            refcount,
-            has_parent_ref,
-            cancel_state,
-        }
-    }
-
-    /// Whether this `CancellationTokenState` is still referenced by any
-    /// `CancellationToken`.
-    fn has_refs(&self) -> bool {
-        self.refcount != 0 || self.has_parent_ref
-    }
-}
-
-/// The maximum permitted amount of references to a CancellationToken. This
-/// is derived from the intent to never use more than 32bit in the `Snapshot`.
-const MAX_REFS: u32 = (std::u32::MAX - 7) >> 3;
-
-/// Internal state of the `CancellationToken` pair above
-struct CancellationTokenState {
-    state: AtomicUsize,
-    parent: Option<NonNull<CancellationTokenState>>,
-    from_parent: SynchronizedThroughParent,
-    synchronized: Mutex<SynchronizedState>,
-}
-
-impl CancellationTokenState {
-    fn new(
-        parent: Option<NonNull<CancellationTokenState>>,
-        state: StateSnapshot,
-    ) -> CancellationTokenState {
-        CancellationTokenState {
-            parent,
-            from_parent: SynchronizedThroughParent {
-                prev_peer: None,
-                next_peer: None,
-            },
-            state: AtomicUsize::new(state.pack()),
-            synchronized: Mutex::new(SynchronizedState::new()),
-        }
-    }
-
-    /// Returns a snapshot of the current atomic state of the token
-    fn snapshot(&self) -> StateSnapshot {
-        StateSnapshot::unpack(self.state.load(Ordering::SeqCst))
-    }
-
-    fn atomic_update_state<F>(&self, mut current_state: StateSnapshot, func: F) -> StateSnapshot
-    where
-        F: Fn(StateSnapshot) -> StateSnapshot,
-    {
-        let mut current_packed_state = current_state.pack();
+        let mut this = self.project();
         loop {
-            let next_state = func(current_state);
-            match self.state.compare_exchange(
-                current_packed_state,
-                next_state.pack(),
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => {
-                    return next_state;
-                }
-                Err(actual) => {
-                    current_packed_state = actual;
-                    current_state = StateSnapshot::unpack(actual);
-                }
-            }
-        }
-    }
-
-    fn increment_refcount(&self, current_state: StateSnapshot) -> StateSnapshot {
-        self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            if state.refcount >= MAX_REFS as usize {
-                eprintln!("[ERROR] Maximum reference count for CancellationToken was exceeded");
-                std::process::abort();
-            }
-            state.refcount += 1;
-            state
-        })
-    }
-
-    fn decrement_refcount(&self, current_state: StateSnapshot) -> StateSnapshot {
-        let current_state = self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            state.refcount -= 1;
-            state
-        });
-
-        // Drop the State if it is not referenced anymore
-        if !current_state.has_refs() {
-            // Safety: `CancellationTokenState` is always stored in refcounted
-            // Boxes
-            let _ = unsafe { Box::from_raw(self as *const Self as *mut Self) };
-        }
-
-        current_state
-    }
-
-    fn remove_parent_ref(&self, current_state: StateSnapshot) -> StateSnapshot {
-        let current_state = self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            state.has_parent_ref = false;
-            state
-        });
-
-        // Drop the State if it is not referenced anymore
-        if !current_state.has_refs() {
-            // Safety: `CancellationTokenState` is always stored in refcounted
-            // Boxes
-            let _ = unsafe { Box::from_raw(self as *const Self as *mut Self) };
-        }
-
-        current_state
-    }
-
-    /// Unregisters a child from the parent token.
-    /// The child tokens state is not exactly known at this point in time.
-    /// If the parent token is cancelled, the child token gets removed from the
-    /// parents list, and might therefore already have been freed. If the parent
-    /// token is not cancelled, the child token is still valid.
-    fn unregister_child(
-        &mut self,
-        mut child_state: NonNull<CancellationTokenState>,
-        current_child_state: StateSnapshot,
-    ) {
-        let removed_child = {
-            // Remove the child toke from the parents linked list
-            let mut guard = self.synchronized.lock().unwrap();
-            if !guard.is_cancelled {
-                // Safety: Since the token was not cancelled, the child must
-                // still be in the list and valid.
-                let mut child_state = unsafe { child_state.as_mut() };
-                debug_assert!(child_state.snapshot().has_parent_ref);
-
-                if guard.first_child == Some(child_state.into()) {
-                    guard.first_child = child_state.from_parent.next_peer;
-                }
-                // Safety: If peers wouldn't be valid anymore, they would try
-                // to remove themselves from the list. This would require locking
-                // the Mutex that we currently own.
-                unsafe {
-                    if let Some(mut prev_peer) = child_state.from_parent.prev_peer {
-                        prev_peer.as_mut().from_parent.next_peer =
-                            child_state.from_parent.next_peer;
-                    }
-                    if let Some(mut next_peer) = child_state.from_parent.next_peer {
-                        next_peer.as_mut().from_parent.prev_peer =
-                            child_state.from_parent.prev_peer;
-                    }
-                }
-                child_state.from_parent.prev_peer = None;
-                child_state.from_parent.next_peer = None;
-
-                // The child is no longer referenced by the parent, since we were able
-                // to remove its reference from the parents list.
-                true
-            } else {
-                // Do not touch the linked list anymore. If the parent is cancelled
-                // it will move all childs outside of the Mutex and manipulate
-                // the pointers there. Manipulating the pointers here too could
-                // lead to races. Therefore leave them just as as and let the
-                // parent deal with it. The parent will make sure to retain a
-                // reference to this state as long as it manipulates the list
-                // pointers. Therefore the pointers are not dangling.
-                false
-            }
-        };
-
-        if removed_child {
-            // If the token removed itself from the parents list, it can reset
-            // the parent ref status. If it is isn't able to do so, because the
-            // parent removed it from the list, there is no need to do this.
-            // The parent ref acts as as another reference count. Therefore
-            // removing this reference can free the object.
-            // Safety: The token was in the list. This means the parent wasn't
-            // cancelled before, and the token must still be alive.
-            unsafe { child_state.as_mut().remove_parent_ref(current_child_state) };
-        }
-
-        // Decrement the refcount on the parent and free it if necessary
-        self.decrement_refcount(self.snapshot());
-    }
-
-    fn cancel(&self) {
-        // Move the state of the CancellationToken from `NotCancelled` to `Cancelling`
-        let mut current_state = self.snapshot();
-
-        let state_after_cancellation = loop {
-            if current_state.cancel_state != CancellationState::NotCancelled {
-                // Another task already initiated the cancellation
-                return;
+            if this.cancellation_token.is_cancelled() {
+                return Poll::Ready(());
             }
 
-            let mut next_state = current_state;
-            next_state.cancel_state = CancellationState::Cancelling;
-            match self.state.compare_exchange(
-                current_state.pack(),
-                next_state.pack(),
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => break next_state,
-                Err(actual) => current_state = StateSnapshot::unpack(actual),
+            // No wakeups can be lost here because there is always a call to
+            // `is_cancelled` between the creation of the future and the call to
+            // `poll`, and the code that sets the cancelled flag does so before
+            // waking the `Notified`.
+            if this.future.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
             }
-        };
 
-        // This task cancelled the token
-
-        // Take the task list out of the Token
-        // We do not want to cancel child token inside this lock. If one of the
-        // child tasks would have additional child tokens, we would recursively
-        // take locks.
-
-        // Doing this action has an impact if the child token is dropped concurrently:
-        // It will try to deregister itself from the parent task, but can not find
-        // itself in the task list anymore. Therefore it needs to assume the parent
-        // has extracted the list and will process it. It may not modify the list.
-        // This is OK from a memory safety perspective, since the parent still
-        // retains a reference to the child task until it finished iterating over
-        // it.
-
-        let mut first_child = {
-            let mut guard = self.synchronized.lock().unwrap();
-            // Save the cancellation also inside the Mutex
-            // This allows child tokens which want to detach themselves to detect
-            // that this is no longer required since the parent cleared the list.
-            guard.is_cancelled = true;
-
-            // Wakeup all waiters
-            // This happens inside the lock to make cancellation reliable
-            // If we would access waiters outside of the lock, the pointers
-            // may no longer be valid.
-            // Typically this shouldn't be an issue, since waking a task should
-            // only move it from the blocked into the ready state and not have
-            // further side effects.
-
-            // Use a reverse iterator, so that the oldest waiter gets
-            // scheduled first
-            guard.waiters.reverse_drain(|waiter| {
-                // We are not allowed to move the `Waker` out of the list node.
-                // The `Future` relies on the fact that the old `Waker` stays there
-                // as long as the `Future` has not completed in order to perform
-                // the `will_wake()` check.
-                // Therefore `wake_by_ref` is used instead of `wake()`
-                if let Some(handle) = &mut waiter.task {
-                    handle.wake_by_ref();
-                }
-                // Mark the waiter to have been removed from the list.
-                waiter.state = PollState::Done;
-            });
-
-            guard.first_child.take()
-        };
-
-        while let Some(mut child) = first_child {
-            // Safety: We know this is a valid pointer since it is in our child pointer
-            // list. It can't have been freed in between, since we retain a a reference
-            // to each child.
-            let mut_child = unsafe { child.as_mut() };
-
-            // Get the next child and clean up list pointers
-            first_child = mut_child.from_parent.next_peer;
-            mut_child.from_parent.prev_peer = None;
-            mut_child.from_parent.next_peer = None;
-
-            // Cancel the child task
-            mut_child.cancel();
-
-            // Drop the parent reference. This `CancellationToken` is not interested
-            // in interacting with the child anymore.
-            // This is ONLY allowed once we promised not to touch the state anymore
-            // after this interaction.
-            mut_child.remove_parent_ref(mut_child.snapshot());
+            this.future.set(this.cancellation_token.inner.notified());
         }
-
-        // The cancellation has completed
-        // At this point in time tasks which registered a wait node can be sure
-        // that this wait node already had been dequeued from the list without
-        // needing to inspect the list.
-        self.atomic_update_state(state_after_cancellation, |mut state| {
-            state.cancel_state = CancellationState::Cancelled;
-            state
-        });
-    }
-
-    /// Returns `true` if the `CancellationToken` had been cancelled
-    fn is_cancelled(&self) -> bool {
-        let current_state = self.snapshot();
-        current_state.cancel_state != CancellationState::NotCancelled
-    }
-
-    /// Registers a waiting task at the `CancellationToken`.
-    /// Safety: This method is only safe as long as the waiting waiting task
-    /// will properly unregister the wait node before it gets moved.
-    unsafe fn register(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        debug_assert_eq!(PollState::New, wait_node.state);
-        let current_state = self.snapshot();
-
-        // Perform an optimistic cancellation check before. This is not strictly
-        // necessary since we also check for cancellation in the Mutex, but
-        // reduces the necessary work to be performed for tasks which already
-        // had been cancelled.
-        if current_state.cancel_state != CancellationState::NotCancelled {
-            return Poll::Ready(());
-        }
-
-        // So far the token is not cancelled. However it could be cancelled before
-        // we get the chance to store the `Waker`. Therefore we need to check
-        // for cancellation again inside the mutex.
-        let mut guard = self.synchronized.lock().unwrap();
-        if guard.is_cancelled {
-            // Cancellation was signalled
-            wait_node.state = PollState::Done;
-            Poll::Ready(())
-        } else {
-            // Added the task to the wait queue
-            wait_node.task = Some(cx.waker().clone());
-            wait_node.state = PollState::Waiting;
-            guard.waiters.add_front(wait_node);
-            Poll::Pending
-        }
-    }
-
-    fn check_for_cancellation(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        debug_assert!(
-            wait_node.task.is_some(),
-            "Method can only be called after task had been registered"
-        );
-
-        let current_state = self.snapshot();
-
-        if current_state.cancel_state != CancellationState::NotCancelled {
-            // If the cancellation had been fully completed we know that our `Waker`
-            // is no longer registered at the `CancellationToken`.
-            // Otherwise the cancel call may or may not yet have iterated
-            // through the waiters list and removed the wait nodes.
-            // If it hasn't yet, we need to remove it. Otherwise an attempt to
-            // reuse the `wait_node´ might get freed due to the `WaitForCancellationFuture`
-            // getting dropped before the cancellation had interacted with it.
-            if current_state.cancel_state != CancellationState::Cancelled {
-                self.unregister(wait_node);
-            }
-            Poll::Ready(())
-        } else {
-            // Check if we need to swap the `Waker`. This will make the check more
-            // expensive, since the `Waker` is synchronized through the Mutex.
-            // If we don't need to perform a `Waker` update, an atomic check for
-            // cancellation is sufficient.
-            let need_waker_update = wait_node
-                .task
-                .as_ref()
-                .map(|waker| !waker.will_wake(cx.waker()))
-                .unwrap_or(true);
-
-            if need_waker_update {
-                let guard = self.synchronized.lock().unwrap();
-                if guard.is_cancelled {
-                    // Cancellation was signalled. Since this cancellation signal
-                    // is set inside the Mutex, the old waiter must already have
-                    // been removed from the waiting list
-                    debug_assert_eq!(PollState::Done, wait_node.state);
-                    wait_node.task = None;
-                    Poll::Ready(())
-                } else {
-                    // The WaitForCancellationFuture is already in the queue.
-                    // The CancellationToken can't have been cancelled,
-                    // since this would change the is_cancelled flag inside the mutex.
-                    // Therefore we just have to update the Waker. A follow-up
-                    // cancellation will always use the new waker.
-                    wait_node.task = Some(cx.waker().clone());
-                    Poll::Pending
-                }
-            } else {
-                // Do nothing. If the token gets cancelled, this task will get
-                // woken again and can fetch the cancellation.
-                Poll::Pending
-            }
-        }
-    }
-
-    fn unregister(&self, wait_node: &mut ListNode<WaitQueueEntry>) {
-        debug_assert!(
-            wait_node.task.is_some(),
-            "waiter can not be active without task"
-        );
-
-        let mut guard = self.synchronized.lock().unwrap();
-        // WaitForCancellationFuture only needs to get removed if it has been added to
-        // the wait queue of the CancellationToken.
-        // This has happened in the PollState::Waiting case.
-        if let PollState::Waiting = wait_node.state {
-            // Safety: Due to the state, we know that the node must be part
-            // of the waiter list
-            if !unsafe { guard.waiters.remove(wait_node) } {
-                // Panic if the address isn't found. This can only happen if the contract was
-                // violated, e.g. the WaitQueueEntry got moved after the initial poll.
-                panic!("Future could not be removed from wait queue");
-            }
-            wait_node.state = PollState::Done;
-        }
-        wait_node.task = None;
     }
 }

tokio-util/src/sync/cancellation_token/tree_node.rs

@@ -0,0 +1,373 @@
+//! This mod provides the logic for the inner tree structure of the CancellationToken.
+//!
+//! CancellationTokens are only light handles with references to TreeNode.
+//! All the logic is actually implemented in the TreeNode.
+//!
+//! A TreeNode is part of the cancellation tree and may have one parent and an arbitrary number of
+//! children.
+//!
+//! A TreeNode can receive the request to perform a cancellation through a CancellationToken.
+//! This cancellation request will cancel the node and all of its descendants.
+//!
+//! As soon as a node cannot get cancelled any more (because it was already cancelled or it has no
+//! more CancellationTokens pointing to it any more), it gets removed from the tree, to keep the
+//! tree as small as possible.
+//!
+//! # Invariants
+//!
+//! Those invariants shall be true at any time.
+//!
+//! 1. A node that has no parents and no handles can no longer be cancelled.
+//!     This is important during both cancellation and refcounting.
+//!
+//! 2. If node B *is* or *was* a child of node A, then node B was created *after* node A.
+//!     This is important for deadlock safety, as it is used for lock order.
+//!     Node B can only become the child of node A in two ways:
+//!         - being created with `child_node()`, in which case it is trivially true that
+//!           node A already existed when node B was created
+//!         - being moved A->C->B to A->B because node C was removed in `decrease_handle_refcount()`
+//!           or `cancel()`. In this case the invariant still holds, as B was younger than C, and C
+//!           was younger than A, therefore B is also younger than A.
+//!
+//! 3. If two nodes are both unlocked and node A is the parent of node B, then node B is a child of
+//!    node A. It is important to always restore that invariant before dropping the lock of a node.
+//!
+//! # Deadlock safety
+//!
+//! We always lock in the order of creation time. We can prove this through invariant #2.
+//! Specifically, through invariant #2, we know that we always have to lock a parent
+//! before its child.
+//!
+use crate::loom::sync::{Arc, Mutex, MutexGuard};
+
+/// A node of the cancellation tree structure
+///
+/// The actual data it holds is wrapped inside a mutex for synchronization.
+pub(crate) struct TreeNode {
+    inner: Mutex<Inner>,
+    waker: tokio::sync::Notify,
+}
+impl TreeNode {
+    pub(crate) fn new() -> Self {
+        Self {
+            inner: Mutex::new(Inner {
+                parent: None,
+                parent_idx: 0,
+                children: vec![],
+                is_cancelled: false,
+                num_handles: 1,
+            }),
+            waker: tokio::sync::Notify::new(),
+        }
+    }
+
+    pub(crate) fn notified(&self) -> tokio::sync::futures::Notified<'_> {
+        self.waker.notified()
+    }
+}
+
+/// The data contained inside a TreeNode.
+///
+/// This struct exists so that the data of the node can be wrapped
+/// in a Mutex.
+struct Inner {
+    parent: Option<Arc<TreeNode>>,
+    parent_idx: usize,
+    children: Vec<Arc<TreeNode>>,
+    is_cancelled: bool,
+    num_handles: usize,
+}
+
+/// Returns whether or not the node is cancelled
+pub(crate) fn is_cancelled(node: &Arc<TreeNode>) -> bool {
+    node.inner.lock().unwrap().is_cancelled
+}
+
+/// Creates a child node
+pub(crate) fn child_node(parent: &Arc<TreeNode>) -> Arc<TreeNode> {
+    let mut locked_parent = parent.inner.lock().unwrap();
+
+    // Do not register as child if we are already cancelled.
+    // Cancelled trees can never be uncancelled and therefore
+    // need no connection to parents or children any more.
+    if locked_parent.is_cancelled {
+        return Arc::new(TreeNode {
+            inner: Mutex::new(Inner {
+                parent: None,
+                parent_idx: 0,
+                children: vec![],
+                is_cancelled: true,
+                num_handles: 1,
+            }),
+            waker: tokio::sync::Notify::new(),
+        });
+    }
+
+    let child = Arc::new(TreeNode {
+        inner: Mutex::new(Inner {
+            parent: Some(parent.clone()),
+            parent_idx: locked_parent.children.len(),
+            children: vec![],
+            is_cancelled: false,
+            num_handles: 1,
+        }),
+        waker: tokio::sync::Notify::new(),
+    });
+
+    locked_parent.children.push(child.clone());
+
+    child
+}
+
+/// Disconnects the given parent from all of its children.
+///
+/// Takes a reference to [Inner] to make sure the parent is already locked.
+fn disconnect_children(node: &mut Inner) {
+    for child in std::mem::take(&mut node.children) {
+        let mut locked_child = child.inner.lock().unwrap();
+        locked_child.parent_idx = 0;
+        locked_child.parent = None;
+    }
+}
+
+/// Figures out the parent of the node and locks the node and its parent atomically.
+///
+/// The basic principle of preventing deadlocks in the tree is
+/// that we always lock the parent first, and then the child.
+/// For more info look at *deadlock safety* and *invariant #2*.
+///
+/// Sadly, it's impossible to figure out the parent of a node without
+/// locking it. To then achieve locking order consistency, the node
+/// has to be unlocked before the parent gets locked.
+/// This leaves a small window where we already assume that we know the parent,
+/// but neither the parent nor the node is locked. Therefore, the parent could change.
+///
+/// To prevent that this problem leaks into the rest of the code, it is abstracted
+/// in this function.
+///
+/// The locked child and optionally its locked parent, if a parent exists, get passed
+/// to the `func` argument via (node, None) or (node, Some(parent)).
+fn with_locked_node_and_parent<F, Ret>(node: &Arc<TreeNode>, func: F) -> Ret
+where
+    F: FnOnce(MutexGuard<'_, Inner>, Option<MutexGuard<'_, Inner>>) -> Ret,
+{
+    let mut potential_parent = {
+        let locked_node = node.inner.lock().unwrap();
+        match locked_node.parent.clone() {
+            Some(parent) => parent,
+            // If we locked the node and its parent is `None`, we are in a valid state
+            // and can return.
+            None => return func(locked_node, None),
+        }
+    };
+
+    loop {
+        // Deadlock safety:
+        //
+        // Due to invariant #2, we know that we have to lock the parent first, and then the child.
+        // This is true even if the potential_parent is no longer the current parent or even its
+        // sibling, as the invariant still holds.
+        let locked_parent = potential_parent.inner.lock().unwrap();
+        let locked_node = node.inner.lock().unwrap();
+
+        let actual_parent = match locked_node.parent.clone() {
+            Some(parent) => parent,
+            // If we locked the node and its parent is `None`, we are in a valid state
+            // and can return.
+            None => {
+                // Was the wrong parent, so unlock it before calling `func`
+                drop(locked_parent);
+                return func(locked_node, None);
+            }
+        };
+
+        // Loop until we managed to lock both the node and its parent
+        if Arc::ptr_eq(&actual_parent, &potential_parent) {
+            return func(locked_node, Some(locked_parent));
+        }
+
+        // Drop locked_parent before reassigning to potential_parent,
+        // as potential_parent is borrowed in it
+        drop(locked_node);
+        drop(locked_parent);
+
+        potential_parent = actual_parent;
+    }
+}
+
+/// Moves all children from `node` to `parent`.
+///
+/// `parent` MUST have been a parent of the node when they both got locked,
+/// otherwise there is a potential for a deadlock as invariant #2 would be violated.
+///
+/// To acquire the locks for node and parent, use [with_locked_node_and_parent].
+fn move_children_to_parent(node: &mut Inner, parent: &mut Inner) {
+    // Pre-allocate in the parent, for performance
+    parent.children.reserve(node.children.len());
+
+    for child in std::mem::take(&mut node.children) {
+        {
+            let mut child_locked = child.inner.lock().unwrap();
+            child_locked.parent = node.parent.clone();
+            child_locked.parent_idx = parent.children.len();
+        }
+        parent.children.push(child);
+    }
+}
+
+/// Removes a child from the parent.
+///
+/// `parent` MUST be the parent of `node`.
+/// To acquire the locks for node and parent, use [with_locked_node_and_parent].
+fn remove_child(parent: &mut Inner, mut node: MutexGuard<'_, Inner>) {
+    // Query the position from where to remove a node
+    let pos = node.parent_idx;
+    node.parent = None;
+    node.parent_idx = 0;
+
+    // Unlock node, so that only one child at a time is locked.
+    // Otherwise we would violate the lock order (see 'deadlock safety') as we
+    // don't know the creation order of the child nodes
+    drop(node);
+
+    // If `node` is the last element in the list, we don't need any swapping
+    if parent.children.len() == pos + 1 {
+        parent.children.pop().unwrap();
+    } else {
+        // If `node` is not the last element in the list, we need to
+        // replace it with the last element
+        let replacement_child = parent.children.pop().unwrap();
+        replacement_child.inner.lock().unwrap().parent_idx = pos;
+        parent.children[pos] = replacement_child;
+    }
+
+    let len = parent.children.len();
+    if 4 * len <= parent.children.capacity() {
+        // equal to:
+        //    parent.children.shrink_to(2 * len);
+        // but shrink_to was not yet stabilized in our minimal compatible version
+        let old_children = std::mem::replace(&mut parent.children, Vec::with_capacity(2 * len));
+        parent.children.extend(old_children);
+    }
+}
+
+/// Increases the reference count of handles.
+pub(crate) fn increase_handle_refcount(node: &Arc<TreeNode>) {
+    let mut locked_node = node.inner.lock().unwrap();
+
+    // Once no handles are left over, the node gets detached from the tree.
+    // There should never be a new handle once all handles are dropped.
+    assert!(locked_node.num_handles > 0);
+
+    locked_node.num_handles += 1;
+}
+
+/// Decreases the reference count of handles.
+///
+/// Once no handle is left, we can remove the node from the
+/// tree and connect its parent directly to its children.
+pub(crate) fn decrease_handle_refcount(node: &Arc<TreeNode>) {
+    let num_handles = {
+        let mut locked_node = node.inner.lock().unwrap();
+        locked_node.num_handles -= 1;
+        locked_node.num_handles
+    };
+
+    if num_handles == 0 {
+        with_locked_node_and_parent(node, |mut node, parent| {
+            // Remove the node from the tree
+            match parent {
+                Some(mut parent) => {
+                    // As we want to remove ourselves from the tree,
+                    // we have to move the children to the parent, so that
+                    // they still receive the cancellation event without us.
+                    // Moving them does not violate invariant #1.
+                    move_children_to_parent(&mut node, &mut parent);
+
+                    // Remove the node from the parent
+                    remove_child(&mut parent, node);
+                }
+                None => {
+                    // Due to invariant #1, we can assume that our
+                    // children can no longer be cancelled through us.
+                    // (as we now have neither a parent nor handles)
+                    // Therefore we can disconnect them.
+                    disconnect_children(&mut node);
+                }
+            }
+        });
+    }
+}
+
+/// Cancels a node and its children.
+pub(crate) fn cancel(node: &Arc<TreeNode>) {
+    let mut locked_node = node.inner.lock().unwrap();
+
+    if locked_node.is_cancelled {
+        return;
+    }
+
+    // One by one, adopt grandchildren and then cancel and detach the child
+    while let Some(child) = locked_node.children.pop() {
+        // This can't deadlock because the mutex we are already
+        // holding is the parent of child.
+        let mut locked_child = child.inner.lock().unwrap();
+
+        // Detach the child from node
+        // No need to modify node.children, as the child already got removed with `.pop`
+        locked_child.parent = None;
+        locked_child.parent_idx = 0;
+
+        // If child is already cancelled, detaching is enough
+        if locked_child.is_cancelled {
+            continue;
+        }
+
+        // Cancel or adopt grandchildren
+        while let Some(grandchild) = locked_child.children.pop() {
+            // This can't deadlock because the two mutexes we are already
+            // holding is the parent and grandparent of grandchild.
+            let mut locked_grandchild = grandchild.inner.lock().unwrap();
+
+            // Detach the grandchild
+            locked_grandchild.parent = None;
+            locked_grandchild.parent_idx = 0;
+
+            // If grandchild is already cancelled, detaching is enough
+            if locked_grandchild.is_cancelled {
+                continue;
+            }
+
+            // For performance reasons, only adopt grandchildren that have children.
+            // Otherwise, just cancel them right away, no need for another iteration.
+            if locked_grandchild.children.is_empty() {
+                // Cancel the grandchild
+                locked_grandchild.is_cancelled = true;
+                locked_grandchild.children = Vec::new();
+                drop(locked_grandchild);
+                grandchild.waker.notify_waiters();
+            } else {
+                // Otherwise, adopt grandchild
+                locked_grandchild.parent = Some(node.clone());
+                locked_grandchild.parent_idx = locked_node.children.len();
+                drop(locked_grandchild);
+                locked_node.children.push(grandchild);
+            }
+        }
+
+        // Cancel the child
+        locked_child.is_cancelled = true;
+        locked_child.children = Vec::new();
+        drop(locked_child);
+        child.waker.notify_waiters();
+
+        // Now the child is cancelled and detached and all its children are adopted.
+        // Just continue until all (including adopted) children are cancelled and detached.
+    }
+
+    // Cancel the node itself.
+    locked_node.is_cancelled = true;
+    locked_node.children = Vec::new();
+    drop(locked_node);
+    node.waker.notify_waiters();
+}

tokio-util/src/sync/intrusive_double_linked_list.rs

@@ -1,788 +0,0 @@
-//! An intrusive double linked list of data
-
-#![allow(dead_code, unreachable_pub)]
-
-use core::{
-    marker::PhantomPinned,
-    ops::{Deref, DerefMut},
-    ptr::NonNull,
-};
-
-/// A node which carries data of type `T` and is stored in an intrusive list
-#[derive(Debug)]
-pub struct ListNode<T> {
-    /// The previous node in the list. `None` if there is no previous node.
-    prev: Option<NonNull<ListNode<T>>>,
-    /// The next node in the list. `None` if there is no previous node.
-    next: Option<NonNull<ListNode<T>>>,
-    /// The data which is associated to this list item
-    data: T,
-    /// Prevents `ListNode`s from being `Unpin`. They may never be moved, since
-    /// the list semantics require addresses to be stable.
-    _pin: PhantomPinned,
-}
-
-impl<T> ListNode<T> {
-    /// Creates a new node with the associated data
-    pub fn new(data: T) -> ListNode<T> {
-        Self {
-            prev: None,
-            next: None,
-            data,
-            _pin: PhantomPinned,
-        }
-    }
-}
-
-impl<T> Deref for ListNode<T> {
-    type Target = T;
-
-    fn deref(&self) -> &T {
-        &self.data
-    }
-}
-
-impl<T> DerefMut for ListNode<T> {
-    fn deref_mut(&mut self) -> &mut T {
-        &mut self.data
-    }
-}
-
-/// An intrusive linked list of nodes, where each node carries associated data
-/// of type `T`.
-#[derive(Debug)]
-pub struct LinkedList<T> {
-    head: Option<NonNull<ListNode<T>>>,
-    tail: Option<NonNull<ListNode<T>>>,
-}
-
-impl<T> LinkedList<T> {
-    /// Creates an empty linked list
-    pub fn new() -> Self {
-        LinkedList::<T> {
-            head: None,
-            tail: None,
-        }
-    }
-
-    /// Adds a node at the front of the linked list.
-    /// Safety: This function is only safe as long as `node` is guaranteed to
-    /// get removed from the list before it gets moved or dropped.
-    /// In addition to this `node` may not be added to another other list before
-    /// it is removed from the current one.
-    pub unsafe fn add_front(&mut self, node: &mut ListNode<T>) {
-        node.next = self.head;
-        node.prev = None;
-        if let Some(mut head) = self.head {
-            head.as_mut().prev = Some(node.into())
-        };
-        self.head = Some(node.into());
-        if self.tail.is_none() {
-            self.tail = Some(node.into());
-        }
-    }
-
-    /// Inserts a node into the list in a way that the list keeps being sorted.
-    /// Safety: This function is only safe as long as `node` is guaranteed to
-    /// get removed from the list before it gets moved or dropped.
-    /// In addition to this `node` may not be added to another other list before
-    /// it is removed from the current one.
-    pub unsafe fn add_sorted(&mut self, node: &mut ListNode<T>)
-    where
-        T: PartialOrd,
-    {
-        if self.head.is_none() {
-            // First node in the list
-            self.head = Some(node.into());
-            self.tail = Some(node.into());
-            return;
-        }
-
-        let mut prev: Option<NonNull<ListNode<T>>> = None;
-        let mut current = self.head;
-
-        while let Some(mut current_node) = current {
-            if node.data < current_node.as_ref().data {
-                // Need to insert before the current node
-                current_node.as_mut().prev = Some(node.into());
-                match prev {
-                    Some(mut prev) => {
-                        prev.as_mut().next = Some(node.into());
-                    }
-                    None => {
-                        // We are inserting at the beginning of the list
-                        self.head = Some(node.into());
-                    }
-                }
-                node.next = current;
-                node.prev = prev;
-                return;
-            }
-            prev = current;
-            current = current_node.as_ref().next;
-        }
-
-        // We looped through the whole list and the nodes data is bigger or equal
-        // than everything we found up to now.
-        // Insert at the end. Since we checked before that the list isn't empty,
-        // tail always has a value.
-        node.prev = self.tail;
-        node.next = None;
-        self.tail.as_mut().unwrap().as_mut().next = Some(node.into());
-        self.tail = Some(node.into());
-    }
-
-    /// Returns the first node in the linked list without removing it from the list
-    /// The function is only safe as long as valid pointers are stored inside
-    /// the linked list.
-    /// The returned pointer is only guaranteed to be valid as long as the list
-    /// is not mutated
-    pub fn peek_first(&self) -> Option<&mut ListNode<T>> {
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list.
-        // The returned node has a pointer which constrains it to the lifetime
-        // of the list. This is ok, since the Node is supposed to outlive
-        // its insertion in the list.
-        unsafe {
-            self.head
-                .map(|mut node| &mut *(node.as_mut() as *mut ListNode<T>))
-        }
-    }
-
-    /// Returns the last node in the linked list without removing it from the list
-    /// The function is only safe as long as valid pointers are stored inside
-    /// the linked list.
-    /// The returned pointer is only guaranteed to be valid as long as the list
-    /// is not mutated
-    pub fn peek_last(&self) -> Option<&mut ListNode<T>> {
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list.
-        // The returned node has a pointer which constrains it to the lifetime
-        // of the list. This is ok, since the Node is supposed to outlive
-        // its insertion in the list.
-        unsafe {
-            self.tail
-                .map(|mut node| &mut *(node.as_mut() as *mut ListNode<T>))
-        }
-    }
-
-    /// Removes the first node from the linked list
-    pub fn remove_first(&mut self) -> Option<&mut ListNode<T>> {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list
-        unsafe {
-            let mut head = self.head?;
-            self.head = head.as_mut().next;
-
-            let first_ref = head.as_mut();
-            match first_ref.next {
-                None => {
-                    // This was the only node in the list
-                    debug_assert_eq!(Some(first_ref.into()), self.tail);
-                    self.tail = None;
-                }
-                Some(mut next) => {
-                    next.as_mut().prev = None;
-                }
-            }
-
-            first_ref.prev = None;
-            first_ref.next = None;
-            Some(&mut *(first_ref as *mut ListNode<T>))
-        }
-    }
-
-    /// Removes the last node from the linked list and returns it
-    pub fn remove_last(&mut self) -> Option<&mut ListNode<T>> {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list
-        unsafe {
-            let mut tail = self.tail?;
-            self.tail = tail.as_mut().prev;
-
-            let last_ref = tail.as_mut();
-            match last_ref.prev {
-                None => {
-                    // This was the last node in the list
-                    debug_assert_eq!(Some(last_ref.into()), self.head);
-                    self.head = None;
-                }
-                Some(mut prev) => {
-                    prev.as_mut().next = None;
-                }
-            }
-
-            last_ref.prev = None;
-            last_ref.next = None;
-            Some(&mut *(last_ref as *mut ListNode<T>))
-        }
-    }
-
-    /// Returns whether the linked list does not contain any node
-    pub fn is_empty(&self) -> bool {
-        if self.head.is_some() {
-            return false;
-        }
-
-        debug_assert!(self.tail.is_none());
-        true
-    }
-
-    /// Removes the given `node` from the linked list.
-    /// Returns whether the `node` was removed.
-    /// It is also only safe if it is known that the `node` is either part of this
-    /// list, or of no list at all. If `node` is part of another list, the
-    /// behavior is undefined.
-    pub unsafe fn remove(&mut self, node: &mut ListNode<T>) -> bool {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        match node.prev {
-            None => {
-                // This might be the first node in the list. If it is not, the
-                // node is not in the list at all. Since our precondition is that
-                // the node must either be in this list or in no list, we check that
-                // the node is really in no list.
-                if self.head != Some(node.into()) {
-                    debug_assert!(node.next.is_none());
-                    return false;
-                }
-                self.head = node.next;
-            }
-            Some(mut prev) => {
-                debug_assert_eq!(prev.as_ref().next, Some(node.into()));
-                prev.as_mut().next = node.next;
-            }
-        }
-
-        match node.next {
-            None => {
-                // This must be the last node in our list. Otherwise the list
-                // is inconsistent.
-                debug_assert_eq!(self.tail, Some(node.into()));
-                self.tail = node.prev;
-            }
-            Some(mut next) => {
-                debug_assert_eq!(next.as_mut().prev, Some(node.into()));
-                next.as_mut().prev = node.prev;
-            }
-        }
-
-        node.next = None;
-        node.prev = None;
-
-        true
-    }
-
-    /// Drains the list iby calling a callback on each list node
-    ///
-    /// The method does not return an iterator since stopping or deferring
-    /// draining the list is not permitted. If the method would push nodes to
-    /// an iterator we could not guarantee that the nodes do not get utilized
-    /// after having been removed from the list anymore.
-    pub fn drain<F>(&mut self, mut func: F)
-    where
-        F: FnMut(&mut ListNode<T>),
-    {
-        let mut current = self.head;
-        self.head = None;
-        self.tail = None;
-
-        while let Some(mut node) = current {
-            // Safety: The nodes have not been removed from the list yet and must
-            // therefore contain valid data. The nodes can also not be added to
-            // the list again during iteration, since the list is mutably borrowed.
-            unsafe {
-                let node_ref = node.as_mut();
-                current = node_ref.next;
-
-                node_ref.next = None;
-                node_ref.prev = None;
-
-                // Note: We do not reset the pointers from the next element in the
-                // list to the current one since we will iterate over the whole
-                // list anyway, and therefore clean up all pointers.
-
-                func(node_ref);
-            }
-        }
-    }
-
-    /// Drains the list in reverse order by calling a callback on each list node
-    ///
-    /// The method does not return an iterator since stopping or deferring
-    /// draining the list is not permitted. If the method would push nodes to
-    /// an iterator we could not guarantee that the nodes do not get utilized
-    /// after having been removed from the list anymore.
-    pub fn reverse_drain<F>(&mut self, mut func: F)
-    where
-        F: FnMut(&mut ListNode<T>),
-    {
-        let mut current = self.tail;
-        self.head = None;
-        self.tail = None;
-
-        while let Some(mut node) = current {
-            // Safety: The nodes have not been removed from the list yet and must
-            // therefore contain valid data. The nodes can also not be added to
-            // the list again during iteration, since the list is mutably borrowed.
-            unsafe {
-                let node_ref = node.as_mut();
-                current = node_ref.prev;
-
-                node_ref.next = None;
-                node_ref.prev = None;
-
-                // Note: We do not reset the pointers from the next element in the
-                // list to the current one since we will iterate over the whole
-                // list anyway, and therefore clean up all pointers.
-
-                func(node_ref);
-            }
-        }
-    }
-}
-
-#[cfg(all(test, feature = "std"))] // Tests make use of Vec at the moment
-mod tests {
-    use super::*;
-
-    fn collect_list<T: Copy>(mut list: LinkedList<T>) -> Vec<T> {
-        let mut result = Vec::new();
-        list.drain(|node| {
-            result.push(**node);
-        });
-        result
-    }
-
-    fn collect_reverse_list<T: Copy>(mut list: LinkedList<T>) -> Vec<T> {
-        let mut result = Vec::new();
-        list.reverse_drain(|node| {
-            result.push(**node);
-        });
-        result
-    }
-
-    unsafe fn add_nodes(list: &mut LinkedList<i32>, nodes: &mut [&mut ListNode<i32>]) {
-        for node in nodes.iter_mut() {
-            list.add_front(node);
-        }
-    }
-
-    unsafe fn assert_clean<T>(node: &mut ListNode<T>) {
-        assert!(node.next.is_none());
-        assert!(node.prev.is_none());
-    }
-
-    #[test]
-    fn insert_and_iterate() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut setup = |list: &mut LinkedList<i32>| {
-                assert_eq!(true, list.is_empty());
-                list.add_front(&mut c);
-                assert_eq!(31, **list.peek_first().unwrap());
-                assert_eq!(false, list.is_empty());
-                list.add_front(&mut b);
-                assert_eq!(7, **list.peek_first().unwrap());
-                list.add_front(&mut a);
-                assert_eq!(5, **list.peek_first().unwrap());
-            };
-
-            let mut list = LinkedList::new();
-            setup(&mut list);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            setup(&mut list);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([31, 7, 5].to_vec(), items);
-        }
-    }
-
-    #[test]
-    fn add_sorted() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-            let mut d = ListNode::new(99);
-
-            let mut list = LinkedList::new();
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut b]);
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut b]);
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut a]);
-            list.add_sorted(&mut b);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut a]);
-            list.add_sorted(&mut b);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut b, &mut a]);
-            list.add_sorted(&mut c);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut b, &mut a]);
-            list.add_sorted(&mut c);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            list.add_sorted(&mut d);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            list.add_sorted(&mut d);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-        }
-    }
-
-    #[test]
-    fn drain_and_collect() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-
-            let taken_items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31].to_vec(), taken_items);
-        }
-    }
-
-    #[test]
-    fn peek_last() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-
-            let last = list.peek_last();
-            assert_eq!(31, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert_eq!(7, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert_eq!(5, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert!(last.is_none());
-        }
-    }
-
-    #[test]
-    fn remove_first() {
-        unsafe {
-            // We iterate forward and backwards through the manipulated lists
-            // to make sure pointers in both directions are still ok.
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([7, 31].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([31, 7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert!(items.is_empty());
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert!(items.is_empty());
-        }
-    }
-
-    #[test]
-    fn remove_last() {
-        unsafe {
-            // We iterate forward and backwards through the manipulated lists
-            // to make sure pointers in both directions are still ok.
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert!(items.is_empty());
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert!(items.is_empty());
-        }
-    }
-
-    #[test]
-    fn remove_by_address() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            {
-                // Remove first
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut c).into()), b.next);
-                assert_eq!(Some((&mut b).into()), c.prev);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([7, 31].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut c).into()), b.next);
-                assert_eq!(Some((&mut b).into()), c.prev);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([31, 7].to_vec(), items);
-            }
-
-            {
-                // Remove middle
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut c).into()), a.next);
-                assert_eq!(Some((&mut a).into()), c.prev);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5, 31].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut c).into()), a.next);
-                assert_eq!(Some((&mut a).into()), c.prev);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([31, 5].to_vec(), items);
-            }
-
-            {
-                // Remove last
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut c));
-                assert_clean((&mut c).into());
-                assert!(b.next.is_none());
-                assert_eq!(Some((&mut b).into()), list.tail);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5, 7].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut c));
-                assert_clean((&mut c).into());
-                assert!(b.next.is_none());
-                assert_eq!(Some((&mut b).into()), list.tail);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([7, 5].to_vec(), items);
-            }
-
-            {
-                // Remove first of two
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut b).into()), list.tail);
-                assert!(b.next.is_none());
-                assert!(b.prev.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([7].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut b).into()), list.tail);
-                assert!(b.next.is_none());
-                assert!(b.prev.is_none());
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([7].to_vec(), items);
-            }
-
-            {
-                // Remove last of two
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut a).into()), list.head);
-                assert_eq!(Some((&mut a).into()), list.tail);
-                assert!(a.next.is_none());
-                assert!(a.prev.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut a).into()), list.head);
-                assert_eq!(Some((&mut a).into()), list.tail);
-                assert!(a.next.is_none());
-                assert!(a.prev.is_none());
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([5].to_vec(), items);
-            }
-
-            {
-                // Remove last item
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                assert!(list.head.is_none());
-                assert!(list.tail.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert!(items.is_empty());
-            }
-
-            {
-                // Remove missing
-                let mut list = LinkedList::new();
-                list.add_front(&mut b);
-                list.add_front(&mut a);
-                assert_eq!(false, list.remove(&mut c));
-            }
-        }
-    }
-}

tokio-util/src/sync/mod.rs

@@ -3,8 +3,6 @@
 mod cancellation_token;
 pub use cancellation_token::{guard::DropGuard, CancellationToken, WaitForCancellationFuture};
 
-mod intrusive_double_linked_list;
-
 mod mpsc;
 pub use mpsc::{PollSendError, PollSender};
 

tokio-util/src/sync/mpsc.rs

@@ -136,6 +136,7 @@ impl<T: Send + 'static> PollSender<T> {
     ///
     /// If `poll_reserve` was not successfully called prior to calling `send_item`, then this method
     /// will panic.
+    #[track_caller]
     pub fn send_item(&mut self, value: T) -> Result<(), PollSendError<T>> {
         let (result, next_state) = match self.take_state() {
             State::Idle(_) | State::Acquiring => {

tokio-util/src/sync/reusable_box.rs

@@ -1,17 +1,18 @@
 use std::alloc::Layout;
+use std::fmt;
 use std::future::Future;
-use std::panic::AssertUnwindSafe;
+use std::marker::PhantomData;
+use std::mem::{self, ManuallyDrop};
 use std::pin::Pin;
-use std::ptr::{self, NonNull};
+use std::ptr;
 use std::task::{Context, Poll};
-use std::{fmt, panic};
 
 /// A reusable `Pin<Box<dyn Future<Output = T> + Send + 'a>>`.
 ///
 /// This type lets you replace the future stored in the box without
 /// reallocating when the size and alignment permits this.
 pub struct ReusableBoxFuture<'a, T> {
-    boxed: NonNull<dyn Future<Output = T> + Send + 'a>,
+    boxed: Pin<Box<dyn Future<Output = T> + Send + 'a>>,
 }
 
 impl<'a, T> ReusableBoxFuture<'a, T> {
@@ -20,11 +21,9 @@ impl<'a, T> ReusableBoxFuture<'a, T> {
     where
         F: Future<Output = T> + Send + 'a,
     {
-        let boxed: Box<dyn Future<Output = T> + Send + 'a> = Box::new(future);
-
-        let boxed = NonNull::from(Box::leak(boxed));
-
-        Self { boxed }
+        Self {
+            boxed: Box::pin(future),
+        }
     }
 
     /// Replace the future currently stored in this box.
@@ -49,62 +48,29 @@ impl<'a, T> ReusableBoxFuture<'a, T> {
     where
         F: Future<Output = T> + Send + 'a,
     {
-        // SAFETY: The pointer is not dangling.
-        let self_layout = {
-            let dyn_future: &(dyn Future<Output = T> + Send) = unsafe { self.boxed.as_ref() };
-            Layout::for_value(dyn_future)
-        };
-
-        if Layout::new::<F>() == self_layout {
-            // SAFETY: We just checked that the layout of F is correct.
-            unsafe {
-                self.set_same_layout(future);
-            }
-
-            Ok(())
-        } else {
-            Err(future)
+        // If we try to inline the contents of this function, the type checker complains because
+        // the bound `T: 'a` is not satisfied in the call to `pending()`. But by putting it in an
+        // inner function that doesn't have `T` as a generic parameter, we implicitly get the bound
+        // `F::Output: 'a` transitively through `F: 'a`, allowing us to call `pending()`.
+        #[inline(always)]
+        fn real_try_set<'a, F>(
+            this: &mut ReusableBoxFuture<'a, F::Output>,
+            future: F,
+        ) -> Result<(), F>
+        where
+            F: Future + Send + 'a,
+        {
+            // future::Pending<T> is a ZST so this never allocates.
+            let boxed = mem::replace(&mut this.boxed, Box::pin(Pending(PhantomData)));
+            reuse_pin_box(boxed, future, |boxed| this.boxed = Pin::from(boxed))
         }
-    }
 
-    /// Set the current future.
-    ///
-    /// # Safety
-    ///
-    /// This function requires that the layout of the provided future is the
-    /// same as `self.layout`.
-    unsafe fn set_same_layout<F>(&mut self, future: F)
-    where
-        F: Future<Output = T> + Send + 'a,
-    {
-        // Drop the existing future, catching any panics.
-        let result = panic::catch_unwind(AssertUnwindSafe(|| {
-            ptr::drop_in_place(self.boxed.as_ptr());
-        }));
-
-        // Overwrite the future behind the pointer. This is safe because the
-        // allocation was allocated with the same size and alignment as the type F.
-        let self_ptr: *mut F = self.boxed.as_ptr() as *mut F;
-        ptr::write(self_ptr, future);
-
-        // Update the vtable of self.boxed. The pointer is not null because we
-        // just got it from self.boxed, which is not null.
-        self.boxed = NonNull::new_unchecked(self_ptr);
-
-        // If the old future's destructor panicked, resume unwinding.
-        match result {
-            Ok(()) => {}
-            Err(payload) => {
-                panic::resume_unwind(payload);
-            }
-        }
+        real_try_set(self, future)
     }
 
     /// Get a pinned reference to the underlying future.
     pub fn get_pin(&mut self) -> Pin<&mut (dyn Future<Output = T> + Send)> {
-        // SAFETY: The user of this box cannot move the box, and we do not move it
-        // either.
-        unsafe { Pin::new_unchecked(self.boxed.as_mut()) }
+        self.boxed.as_mut()
     }
 
     /// Poll the future stored inside this box.
@@ -122,27 +88,84 @@ impl<T> Future for ReusableBoxFuture<'_, T> {
     }
 }
 
-// The future stored inside ReusableBoxFuture<'_, T> must be Send.
-unsafe impl<T> Send for ReusableBoxFuture<'_, T> {}
-
 // The only method called on self.boxed is poll, which takes &mut self, so this
 // struct being Sync does not permit any invalid access to the Future, even if
 // the future is not Sync.
 unsafe impl<T> Sync for ReusableBoxFuture<'_, T> {}
 
-// Just like a Pin<Box<dyn Future>> is always Unpin, so is this type.
-impl<T> Unpin for ReusableBoxFuture<'_, T> {}
-
-impl<T> Drop for ReusableBoxFuture<'_, T> {
-    fn drop(&mut self) {
-        unsafe {
-            drop(Box::from_raw(self.boxed.as_ptr()));
-        }
-    }
-}
-
 impl<T> fmt::Debug for ReusableBoxFuture<'_, T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("ReusableBoxFuture").finish()
     }
 }
+
+fn reuse_pin_box<T: ?Sized, U, O, F>(boxed: Pin<Box<T>>, new_value: U, callback: F) -> Result<O, U>
+where
+    F: FnOnce(Box<U>) -> O,
+{
+    let layout = Layout::for_value::<T>(&*boxed);
+    if layout != Layout::new::<U>() {
+        return Err(new_value);
+    }
+
+    // SAFETY: We don't ever construct a non-pinned reference to the old `T` from now on, and we
+    // always drop the `T`.
+    let raw: *mut T = Box::into_raw(unsafe { Pin::into_inner_unchecked(boxed) });
+
+    // When dropping the old value panics, we still want to call `callback` — so move the rest of
+    // the code into a guard type.
+    let guard = CallOnDrop::new(|| {
+        let raw: *mut U = raw.cast::<U>();
+        unsafe { raw.write(new_value) };
+
+        // SAFETY:
+        // - `T` and `U` have the same layout.
+        // - `raw` comes from a `Box` that uses the same allocator as this one.
+        // - `raw` points to a valid instance of `U` (we just wrote it in).
+        let boxed = unsafe { Box::from_raw(raw) };
+
+        callback(boxed)
+    });
+
+    // Drop the old value.
+    unsafe { ptr::drop_in_place(raw) };
+
+    // Run the rest of the code.
+    Ok(guard.call())
+}
+
+struct CallOnDrop<O, F: FnOnce() -> O> {
+    f: ManuallyDrop<F>,
+}
+
+impl<O, F: FnOnce() -> O> CallOnDrop<O, F> {
+    fn new(f: F) -> Self {
+        let f = ManuallyDrop::new(f);
+        Self { f }
+    }
+    fn call(self) -> O {
+        let mut this = ManuallyDrop::new(self);
+        let f = unsafe { ManuallyDrop::take(&mut this.f) };
+        f()
+    }
+}
+
+impl<O, F: FnOnce() -> O> Drop for CallOnDrop<O, F> {
+    fn drop(&mut self) {
+        let f = unsafe { ManuallyDrop::take(&mut self.f) };
+        f();
+    }
+}
+
+/// The same as `std::future::Pending<T>`; we can't use that type directly because on rustc
+/// versions <1.60 it didn't unconditionally implement `Send`.
+// FIXME: use `std::future::Pending<T>` once the MSRV is >=1.60
+struct Pending<T>(PhantomData<fn() -> T>);
+
+impl<T> Future for Pending<T> {
+    type Output = T;
+
+    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
+        Poll::Pending
+    }
+}

tokio-util/src/sync/tests/loom_cancellation_token.rs

@@ -80,7 +80,7 @@ fn drop_token_no_child() {
 }
 
 #[test]
-fn drop_token_with_childs() {
+fn drop_token_with_children() {
     loom::model(|| {
         let token1 = CancellationToken::new();
         let child_token1 = token1.child_token();

tokio-util/src/task/join_map.rs

@@ -50,9 +50,9 @@ use tokio::task::{AbortHandle, Id, JoinError, JoinSet, LocalSet};
 ///
 ///     let mut seen = [false; 10];
 ///
-///     // When a task completes, `join_one` returns the task's key along
+///     // When a task completes, `join_next` returns the task's key along
 ///     // with its output.
-///     while let Some((key, res)) = map.join_one().await {
+///     while let Some((key, res)) = map.join_next().await {
 ///         seen[key] = true;
 ///         assert!(res.is_ok(), "task {} completed successfully!", key);
 ///     }
@@ -82,7 +82,7 @@ use tokio::task::{AbortHandle, Id, JoinError, JoinSet, LocalSet};
 ///     // provided key.
 ///     assert!(aborted);
 ///
-///     while let Some((key, res)) = map.join_one().await {
+///     while let Some((key, res)) = map.join_next().await {
 ///         if key == "goodbye world" {
 ///             // The aborted task should complete with a cancelled `JoinError`.
 ///             assert!(res.unwrap_err().is_cancelled());
@@ -277,14 +277,15 @@ where
     ///
     /// If a task previously existed in the `JoinMap` for this key, that task
     /// will be cancelled and replaced with the new one. The previous task will
-    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
     /// *not* return a cancelled [`JoinError`] for that task.
     ///
     /// # Panics
     ///
     /// This method panics if called outside of a Tokio runtime.
     ///
-    /// [`join_one`]: Self::join_one
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
     pub fn spawn<F>(&mut self, key: K, task: F)
     where
         F: Future<Output = V>,
@@ -300,10 +301,11 @@ where
     ///
     /// If a task previously existed in the `JoinMap` for this key, that task
     /// will be cancelled and replaced with the new one. The previous task will
-    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
     /// *not* return a cancelled [`JoinError`] for that task.
     ///
-    /// [`join_one`]: Self::join_one
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
     pub fn spawn_on<F>(&mut self, key: K, task: F, handle: &Handle)
     where
         F: Future<Output = V>,
@@ -319,7 +321,7 @@ where
     ///
     /// If a task previously existed in the `JoinMap` for this key, that task
     /// will be cancelled and replaced with the new one. The previous task will
-    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
     /// *not* return a cancelled [`JoinError`] for that task.
     ///
     /// # Panics
@@ -327,7 +329,8 @@ where
     /// This method panics if it is called outside of a `LocalSet`.
     ///
     /// [`LocalSet`]: tokio::task::LocalSet
-    /// [`join_one`]: Self::join_one
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
     pub fn spawn_local<F>(&mut self, key: K, task: F)
     where
         F: Future<Output = V>,
@@ -342,11 +345,12 @@ where
     ///
     /// If a task previously existed in the `JoinMap` for this key, that task
     /// will be cancelled and replaced with the new one. The previous task will
-    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_next`] will
     /// *not* return a cancelled [`JoinError`] for that task.
     ///
     /// [`LocalSet`]: tokio::task::LocalSet
-    /// [`join_one`]: Self::join_one
+    /// [`join_next`]: Self::join_next
+    #[track_caller]
     pub fn spawn_local_on<F>(&mut self, key: K, task: F, local_set: &LocalSet)
     where
         F: Future<Output = V>,
@@ -395,7 +399,7 @@ where
     ///
     /// # Cancel Safety
     ///
-    /// This method is cancel safe. If `join_one` is used as the event in a [`tokio::select!`]
+    /// This method is cancel safe. If `join_next` is used as the event in a [`tokio::select!`]
     /// statement and some other branch completes first, it is guaranteed that no tasks were
     /// removed from this `JoinMap`.
     ///
@@ -412,16 +416,14 @@ where
     ///  * `None` if the `JoinMap` is empty.
     ///
     /// [`tokio::select!`]: tokio::select
-    pub async fn join_one(&mut self) -> Option<(K, Result<V, JoinError>)> {
-        let (res, id) = match self.tasks.join_one_with_id().await {
-            Ok(task) => {
-                let (id, output) = task?;
-                (Ok(output), id)
-            }
-            Err(e) => {
+    pub async fn join_next(&mut self) -> Option<(K, Result<V, JoinError>)> {
+        let (res, id) = match self.tasks.join_next_with_id().await {
+            Some(Ok((id, output))) => (Ok(output), id),
+            Some(Err(e)) => {
                 let id = e.id();
                 (Err(e), id)
             }
+            None => return None,
         };
         let key = self.remove_by_id(id)?;
         Some((key, res))
@@ -429,17 +431,17 @@ where
 
     /// Aborts all tasks and waits for them to finish shutting down.
     ///
-    /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_one`] in
+    /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_next`] in
     /// a loop until it returns `None`.
     ///
     /// This method ignores any panics in the tasks shutting down. When this call returns, the
     /// `JoinMap` will be empty.
     ///
     /// [`abort_all`]: fn@Self::abort_all
-    /// [`join_one`]: fn@Self::join_one
+    /// [`join_next`]: fn@Self::join_next
     pub async fn shutdown(&mut self) {
         self.abort_all();
-        while self.join_one().await.is_some() {}
+        while self.join_next().await.is_some() {}
     }
 
     /// Abort the task corresponding to the provided `key`.
@@ -465,7 +467,7 @@ where
     /// // Look up the "goodbye world" task in the map and abort it.
     /// map.abort("goodbye world");
     ///
-    /// while let Some((key, res)) = map.join_one().await {
+    /// while let Some((key, res)) = map.join_next().await {
     ///     if key == "goodbye world" {
     ///         // The aborted task should complete with a cancelled `JoinError`.
     ///         assert!(res.unwrap_err().is_cancelled());
@@ -546,7 +548,7 @@ where
     /// map.abort_matching(|key| key.starts_with("goodbye"));
     ///
     /// let mut seen = 0;
-    /// while let Some((key, res)) = map.join_one().await {
+    /// while let Some((key, res)) = map.join_next().await {
     ///     seen += 1;
     ///     if key.starts_with("goodbye") {
     ///         // The aborted task should complete with a cancelled `JoinError`.
@@ -565,7 +567,7 @@ where
     pub fn abort_matching(&mut self, mut predicate: impl FnMut(&K) -> bool) {
         // Note: this method iterates over the tasks and keys *without* removing
         // any entries, so that the keys from aborted tasks can still be
-        // returned when calling `join_one` in the future.
+        // returned when calling `join_next` in the future.
         for (Key { ref key, .. }, task) in &self.tasks_by_key {
             if predicate(key) {
                 task.abort();
@@ -576,9 +578,9 @@ where
     /// Returns `true` if this `JoinMap` contains a task for the provided key.
     ///
     /// If the task has completed, but its output hasn't yet been consumed by a
-    /// call to [`join_one`], this method will still return `true`.
+    /// call to [`join_next`], this method will still return `true`.
     ///
-    /// [`join_one`]: fn@Self::join_one
+    /// [`join_next`]: fn@Self::join_next
     pub fn contains_key<Q: ?Sized>(&self, key: &Q) -> bool
     where
         Q: Hash + Eq,
@@ -591,9 +593,9 @@ where
     /// [task ID].
     ///
     /// If the task has completed, but its output hasn't yet been consumed by a
-    /// call to [`join_one`], this method will still return `true`.
+    /// call to [`join_next`], this method will still return `true`.
     ///
-    /// [`join_one`]: fn@Self::join_one
+    /// [`join_next`]: fn@Self::join_next
     /// [task ID]: tokio::task::Id
     pub fn contains_task(&self, task: &Id) -> bool {
         self.get_by_id(task).is_some()
@@ -737,7 +739,7 @@ where
     /// Aborts all tasks on this `JoinMap`.
     ///
     /// This does not remove the tasks from the `JoinMap`. To wait for the tasks to complete
-    /// cancellation, you should call `join_one` in a loop until the `JoinMap` is empty.
+    /// cancellation, you should call `join_next` in a loop until the `JoinMap` is empty.
     pub fn abort_all(&mut self) {
         self.tasks.abort_all()
     }

tokio-util/src/task/mod.rs

@@ -2,7 +2,9 @@
 
 #[cfg(tokio_unstable)]
 mod join_map;
+#[cfg(not(target_os = "wasi"))]
 mod spawn_pinned;
+#[cfg(not(target_os = "wasi"))]
 pub use spawn_pinned::LocalPoolHandle;
 
 #[cfg(tokio_unstable)]

tokio-util/src/task/spawn_pinned.rs

@@ -9,7 +9,44 @@ use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
 use tokio::sync::oneshot;
 use tokio::task::{spawn_local, JoinHandle, LocalSet};
 
-/// A handle to a local pool, used for spawning `!Send` tasks.
+/// A cloneable handle to a local pool, used for spawning `!Send` tasks.
+///
+/// Internally the local pool uses a [`tokio::task::LocalSet`] for each worker thread
+/// in the pool. Consequently you can also use [`tokio::task::spawn_local`] (which will
+/// execute on the same thread) inside the Future you supply to the various spawn methods
+/// of `LocalPoolHandle`,
+///
+/// [`tokio::task::LocalSet`]: tokio::task::LocalSet
+/// [`tokio::task::spawn_local`]: tokio::task::spawn_local
+///
+/// # Examples
+///
+/// ```
+/// use std::rc::Rc;
+/// use tokio::{self, task };
+/// use tokio_util::task::LocalPoolHandle;
+///
+/// #[tokio::main(flavor = "current_thread")]
+/// async fn main() {
+///     let pool = LocalPoolHandle::new(5);
+///
+///     let output = pool.spawn_pinned(|| {
+///         // `data` is !Send + !Sync
+///         let data = Rc::new("local data");
+///         let data_clone = data.clone();
+///
+///         async move {
+///             task::spawn_local(async move {
+///                 println!("{}", data_clone);
+///             });
+///     
+///             data.to_string()
+///         }   
+///     }).await.unwrap();
+///     println!("output: {}", output);
+/// }
+/// ```
+///
 #[derive(Clone)]
 pub struct LocalPoolHandle {
     pool: Arc<LocalPool>,
@@ -20,7 +57,9 @@ impl LocalPoolHandle {
     /// pool via [`LocalPoolHandle::spawn_pinned`].
     ///
     /// # Panics
+    ///
     /// Panics if the pool size is less than one.
+    #[track_caller]
     pub fn new(pool_size: usize) -> LocalPoolHandle {
         assert!(pool_size > 0);
 
@@ -33,6 +72,22 @@ impl LocalPoolHandle {
         LocalPoolHandle { pool }
     }
 
+    /// Returns the number of threads of the Pool.
+    #[inline]
+    pub fn num_threads(&self) -> usize {
+        self.pool.workers.len()
+    }
+
+    /// Returns the number of tasks scheduled on each worker. The indices of the
+    /// worker threads correspond to the indices of the returned `Vec`.
+    pub fn get_task_loads_for_each_worker(&self) -> Vec<usize> {
+        self.pool
+            .workers
+            .iter()
+            .map(|worker| worker.task_count.load(Ordering::SeqCst))
+            .collect::<Vec<_>>()
+    }
+
     /// Spawn a task onto a worker thread and pin it there so it can't be moved
     /// off of the thread. Note that the future is not [`Send`], but the
     /// [`FnOnce`] which creates it is.
@@ -69,7 +124,61 @@ impl LocalPoolHandle {
         Fut: Future + 'static,
         Fut::Output: Send + 'static,
     {
-        self.pool.spawn_pinned(create_task)
+        self.pool
+            .spawn_pinned(create_task, WorkerChoice::LeastBurdened)
+    }
+
+    /// Differs from `spawn_pinned` only in that you can choose a specific worker thread
+    /// of the pool, whereas `spawn_pinned` chooses the worker with the smallest
+    /// number of tasks scheduled.
+    ///
+    /// A worker thread is chosen by index. Indices are 0 based and the largest index
+    /// is given by `num_threads() - 1`
+    ///
+    /// # Panics
+    ///
+    /// This method panics if the index is out of bounds.
+    ///
+    /// # Examples
+    ///
+    /// This method can be used to spawn a task on all worker threads of the pool:
+    ///
+    /// ```
+    /// use tokio_util::task::LocalPoolHandle;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     const NUM_WORKERS: usize = 3;
+    ///     let pool = LocalPoolHandle::new(NUM_WORKERS);
+    ///     let handles = (0..pool.num_threads())
+    ///         .map(|worker_idx| {
+    ///             pool.spawn_pinned_by_idx(
+    ///                 || {
+    ///                     async {
+    ///                         "test"
+    ///                     }
+    ///                 },
+    ///                 worker_idx,
+    ///             )
+    ///         })
+    ///         .collect::<Vec<_>>();
+    ///
+    ///     for handle in handles {
+    ///         handle.await.unwrap();
+    ///     }
+    /// }
+    /// ```
+    ///
+    #[track_caller]
+    pub fn spawn_pinned_by_idx<F, Fut>(&self, create_task: F, idx: usize) -> JoinHandle<Fut::Output>
+    where
+        F: FnOnce() -> Fut,
+        F: Send + 'static,
+        Fut: Future + 'static,
+        Fut::Output: Send + 'static,
+    {
+        self.pool
+            .spawn_pinned(create_task, WorkerChoice::ByIdx(idx))
     }
 }
 
@@ -79,13 +188,23 @@ impl Debug for LocalPoolHandle {
     }
 }
 
+enum WorkerChoice {
+    LeastBurdened,
+    ByIdx(usize),
+}
+
 struct LocalPool {
     workers: Vec<LocalWorkerHandle>,
 }
 
 impl LocalPool {
     /// Spawn a `?Send` future onto a worker
-    fn spawn_pinned<F, Fut>(&self, create_task: F) -> JoinHandle<Fut::Output>
+    #[track_caller]
+    fn spawn_pinned<F, Fut>(
+        &self,
+        create_task: F,
+        worker_choice: WorkerChoice,
+    ) -> JoinHandle<Fut::Output>
     where
         F: FnOnce() -> Fut,
         F: Send + 'static,
@@ -93,8 +212,10 @@ impl LocalPool {
         Fut::Output: Send + 'static,
     {
         let (sender, receiver) = oneshot::channel();
-
-        let (worker, job_guard) = self.find_and_incr_least_burdened_worker();
+        let (worker, job_guard) = match worker_choice {
+            WorkerChoice::LeastBurdened => self.find_and_incr_least_burdened_worker(),
+            WorkerChoice::ByIdx(idx) => self.find_worker_by_idx(idx),
+        };
         let worker_spawner = worker.spawner.clone();
 
         // Spawn a future onto the worker's runtime so we can immediately return
@@ -206,6 +327,14 @@ impl LocalPool {
             }
         }
     }
+
+    #[track_caller]
+    fn find_worker_by_idx(&self, idx: usize) -> (&LocalWorkerHandle, JobCountGuard) {
+        let worker = &self.workers[idx];
+        worker.task_count.fetch_add(1, Ordering::SeqCst);
+
+        (worker, JobCountGuard(Arc::clone(&worker.task_count)))
+    }
 }
 
 /// Automatically decrements a worker's job count when a job finishes (when

tokio-util/src/time/delay_queue.rs

@@ -190,7 +190,7 @@ impl<T> SlabStorage<T> {
         let key_contained = self.key_map.contains_key(&key.into());
 
         if key_contained {
-            // It's possible that a `compact` call creates capacitiy in `self.inner` in
+            // It's possible that a `compact` call creates capacity in `self.inner` in
             // such a way that a `self.inner.insert` call creates a `key` which was
             // previously given out during an `insert` call prior to the `compact` call.
             // If `key` is contained in `self.key_map`, we have encountered this exact situation,
@@ -531,6 +531,7 @@ impl<T> DelayQueue<T> {
     /// [`reset`]: method@Self::reset
     /// [`Key`]: struct@Key
     /// [type]: #
+    #[track_caller]
     pub fn insert_at(&mut self, value: T, when: Instant) -> Key {
         assert!(self.slab.len() < MAX_ENTRIES, "max entries exceeded");
 
@@ -649,10 +650,12 @@ impl<T> DelayQueue<T> {
     /// [`reset`]: method@Self::reset
     /// [`Key`]: struct@Key
     /// [type]: #
+    #[track_caller]
     pub fn insert(&mut self, value: T, timeout: Duration) -> Key {
         self.insert_at(value, Instant::now() + timeout)
     }
 
+    #[track_caller]
     fn insert_idx(&mut self, when: u64, key: Key) {
         use self::wheel::{InsertError, Stack};
 
@@ -674,6 +677,7 @@ impl<T> DelayQueue<T> {
     /// # Panics
     ///
     /// Panics if the key is not contained in the expired queue or the wheel.
+    #[track_caller]
     fn remove_key(&mut self, key: &Key) {
         use crate::time::wheel::Stack;
 
@@ -713,6 +717,7 @@ impl<T> DelayQueue<T> {
     /// assert_eq!(*item.get_ref(), "foo");
     /// # }
     /// ```
+    #[track_caller]
     pub fn remove(&mut self, key: &Key) -> Expired<T> {
         let prev_deadline = self.next_deadline();
 
@@ -769,6 +774,7 @@ impl<T> DelayQueue<T> {
     /// // "foo" is now scheduled to be returned in 10 seconds
     /// # }
     /// ```
+    #[track_caller]
     pub fn reset_at(&mut self, key: &Key, when: Instant) {
         self.remove_key(key);
 
@@ -873,6 +879,7 @@ impl<T> DelayQueue<T> {
     /// // "foo"is now scheduled to be returned in 10 seconds
     /// # }
     /// ```
+    #[track_caller]
     pub fn reset(&mut self, key: &Key, timeout: Duration) {
         self.reset_at(key, Instant::now() + timeout);
     }
@@ -978,7 +985,12 @@ impl<T> DelayQueue<T> {
     /// assert!(delay_queue.capacity() >= 11);
     /// # }
     /// ```
+    #[track_caller]
     pub fn reserve(&mut self, additional: usize) {
+        assert!(
+            self.slab.capacity() + additional <= MAX_ENTRIES,
+            "max queue capacity exceeded"
+        );
         self.slab.reserve(additional);
     }
 
@@ -1117,6 +1129,7 @@ impl<T> wheel::Stack for Stack<T> {
         }
     }
 
+    #[track_caller]
     fn remove(&mut self, item: &Self::Borrowed, store: &mut Self::Store) {
         let key = *item;
         assert!(store.contains(item));

tokio-util/src/time/wheel/mod.rs

@@ -118,6 +118,7 @@ where
     }
 
     /// Remove `item` from the timing wheel.
+    #[track_caller]
     pub(crate) fn remove(&mut self, item: &T::Borrowed, store: &mut T::Store) {
         let when = T::when(item, store);
 

tokio-util/src/udp/frame.rs

@@ -81,17 +81,21 @@ where
             }
 
             // We're out of data. Try and fetch more data to decode
-            let addr = unsafe {
-                // Convert `&mut [MaybeUnit<u8>]` to `&mut [u8]` because we will be
-                // writing to it via `poll_recv_from` and therefore initializing the memory.
-                let buf = &mut *(pin.rd.chunk_mut() as *mut _ as *mut [MaybeUninit<u8>]);
+            let addr = {
+                // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
+                // transparent wrapper around `[MaybeUninit<u8>]`.
+                let buf = unsafe { &mut *(pin.rd.chunk_mut() as *mut _ as *mut [MaybeUninit<u8>]) };
                 let mut read = ReadBuf::uninit(buf);
                 let ptr = read.filled().as_ptr();
                 let res = ready!(pin.socket.borrow().poll_recv_from(cx, &mut read));
 
                 assert_eq!(ptr, read.filled().as_ptr());
                 let addr = res?;
-                pin.rd.advance_mut(read.filled().len());
+
+                // Safety: This is guaranteed to be the number of initialized (and read) bytes due
+                // to the invariants provided by `ReadBuf::filled`.
+                unsafe { pin.rd.advance_mut(read.filled().len()) };
+
                 addr
             };
 

tokio-util/tests/context.rs

@@ -1,4 +1,5 @@
 #![cfg(feature = "rt")]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 #![warn(rust_2018_idioms)]
 
 use tokio::runtime::Builder;

tokio-util/tests/framed_write.rs

@@ -130,7 +130,7 @@ fn write_hits_backpressure() {
             _ => unreachable!(),
         }
 
-        // Push a new new chunk
+        // Push a new chunk
         mock.calls.push_back(Ok(b[..].to_vec()));
     }
     // 1 'wouldblock', 4 * 2KB buffers, 1 b-byte buffer

tokio-util/tests/io_sync_bridge.rs

@@ -1,8 +1,9 @@
 #![cfg(feature = "io-util")]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 
 use std::error::Error;
-use std::io::{Cursor, Read, Result as IoResult};
-use tokio::io::AsyncRead;
+use std::io::{Cursor, Read, Result as IoResult, Write};
+use tokio::io::{AsyncRead, AsyncReadExt};
 use tokio_util::io::SyncIoBridge;
 
 async fn test_reader_len(
@@ -41,3 +42,21 @@ async fn test_async_write_to_sync() -> Result<(), Box<dyn Error>> {
     assert_eq!(dest.as_slice(), src);
     Ok(())
 }
+
+#[tokio::test]
+async fn test_shutdown() -> Result<(), Box<dyn Error>> {
+    let (s1, mut s2) = tokio::io::duplex(1024);
+    let (_rh, wh) = tokio::io::split(s1);
+    tokio::task::spawn_blocking(move || -> std::io::Result<_> {
+        let mut wh = SyncIoBridge::new(wh);
+        wh.write_all(b"hello")?;
+        wh.shutdown()?;
+        assert!(wh.write_all(b" world").is_err());
+        Ok(())
+    })
+    .await??;
+    let mut buf = vec![];
+    s2.read_to_end(&mut buf).await?;
+    assert_eq!(buf, b"hello");
+    Ok(())
+}

tokio-util/tests/panic.rs

@@ -0,0 +1,226 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support panic recovery
+
+use parking_lot::{const_mutex, Mutex};
+use std::error::Error;
+use std::panic;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use tokio::sync::mpsc::channel;
+use tokio::time::{Duration, Instant};
+use tokio_test::task;
+use tokio_util::io::SyncIoBridge;
+use tokio_util::sync::PollSender;
+use tokio_util::task::LocalPoolHandle;
+use tokio_util::time::DelayQueue;
+
+// Taken from tokio-util::time::wheel, if that changes then
+const MAX_DURATION_MS: u64 = (1 << (36)) - 1;
+
+fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<String> {
+    static PANIC_MUTEX: Mutex<()> = const_mutex(());
+
+    {
+        let _guard = PANIC_MUTEX.lock();
+        let panic_file: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
+
+        let prev_hook = panic::take_hook();
+        {
+            let panic_file = panic_file.clone();
+            panic::set_hook(Box::new(move |panic_info| {
+                let panic_location = panic_info.location().unwrap();
+                panic_file
+                    .lock()
+                    .clone_from(&Some(panic_location.file().to_string()));
+            }));
+        }
+
+        let result = panic::catch_unwind(func);
+        // Return to the previously set panic hook (maybe default) so that we get nice error
+        // messages in the tests.
+        panic::set_hook(prev_hook);
+
+        if result.is_err() {
+            panic_file.lock().clone()
+        } else {
+            None
+        }
+    }
+}
+
+#[test]
+fn sync_bridge_new_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let _ = SyncIoBridge::new(tokio::io::empty());
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn poll_sender_send_item_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let (send, _) = channel::<u32>(3);
+        let mut send = PollSender::new(send);
+
+        let _ = send.send_item(42);
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+
+fn local_pool_handle_new_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let _ = LocalPoolHandle::new(0);
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+
+fn local_pool_handle_spawn_pinned_by_idx_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+
+        rt.block_on(async {
+            let handle = LocalPoolHandle::new(2);
+            handle.spawn_pinned_by_idx(|| async { "test" }, 3);
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+#[test]
+fn delay_queue_insert_at_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::with_capacity(3));
+
+            //let st = std::time::Instant::from(SystemTime::UNIX_EPOCH);
+            let _k = queue.insert_at(
+                "1",
+                Instant::now() + Duration::from_millis(MAX_DURATION_MS + 1),
+            );
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn delay_queue_insert_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::with_capacity(3));
+
+            let _k = queue.insert("1", Duration::from_millis(MAX_DURATION_MS + 1));
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn delay_queue_remove_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::with_capacity(3));
+
+            let key = queue.insert_at("1", Instant::now());
+            queue.remove(&key);
+            queue.remove(&key);
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn delay_queue_reset_at_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::with_capacity(3));
+
+            let key = queue.insert_at("1", Instant::now());
+            queue.reset_at(
+                &key,
+                Instant::now() + Duration::from_millis(MAX_DURATION_MS + 1),
+            );
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn delay_queue_reset_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::with_capacity(3));
+
+            let key = queue.insert_at("1", Instant::now());
+            queue.reset(&key, Duration::from_millis(MAX_DURATION_MS + 1));
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+#[test]
+fn delay_queue_reserve_panic_caller() -> Result<(), Box<dyn Error>> {
+    let panic_location_file = test_panic(|| {
+        let rt = basic();
+        rt.block_on(async {
+            let mut queue = task::spawn(DelayQueue::<u32>::with_capacity(3));
+
+            queue.reserve((1 << 30) as usize);
+        });
+    });
+
+    // The panic location should be in this file
+    assert_eq!(&panic_location_file.unwrap(), file!());
+
+    Ok(())
+}
+
+fn basic() -> Runtime {
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+}

tokio-util/tests/reusable_box.rs

@@ -1,10 +1,23 @@
 use futures::future::FutureExt;
 use std::alloc::Layout;
 use std::future::Future;
+use std::marker::PhantomPinned;
 use std::pin::Pin;
+use std::rc::Rc;
 use std::task::{Context, Poll};
 use tokio_util::sync::ReusableBoxFuture;
 
+#[test]
+// Clippy false positive; it's useful to be able to test the trait impls for any lifetime
+#[allow(clippy::extra_unused_lifetimes)]
+fn traits<'a>() {
+    fn assert_traits<T: Send + Sync + Unpin>() {}
+    // Use a type that is !Unpin
+    assert_traits::<ReusableBoxFuture<'a, PhantomPinned>>();
+    // Use a type that is !Send + !Sync
+    assert_traits::<ReusableBoxFuture<'a, Rc<()>>>();
+}
+
 #[test]
 fn test_different_futures() {
     let fut = async move { 10 };

tokio-util/tests/spawn_pinned.rs

@@ -1,7 +1,9 @@
 #![warn(rust_2018_idioms)]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support threads
 
 use std::rc::Rc;
 use std::sync::Arc;
+use tokio::sync::Barrier;
 use tokio_util::task;
 
 /// Simple test of running a !Send future via spawn_pinned
@@ -191,3 +193,45 @@ async fn tasks_are_balanced() {
     // be on separate workers/threads.
     assert_ne!(thread_id1, thread_id2);
 }
+
+#[tokio::test]
+async fn spawn_by_idx() {
+    let pool = task::LocalPoolHandle::new(3);
+    let barrier = Arc::new(Barrier::new(4));
+    let barrier1 = barrier.clone();
+    let barrier2 = barrier.clone();
+    let barrier3 = barrier.clone();
+
+    let handle1 = pool.spawn_pinned_by_idx(
+        || async move {
+            barrier1.wait().await;
+            std::thread::current().id()
+        },
+        0,
+    );
+    let _ = pool.spawn_pinned_by_idx(
+        || async move {
+            barrier2.wait().await;
+            std::thread::current().id()
+        },
+        0,
+    );
+    let handle2 = pool.spawn_pinned_by_idx(
+        || async move {
+            barrier3.wait().await;
+            std::thread::current().id()
+        },
+        1,
+    );
+
+    let loads = pool.get_task_loads_for_each_worker();
+    barrier.wait().await;
+    assert_eq!(loads[0], 2);
+    assert_eq!(loads[1], 1);
+    assert_eq!(loads[2], 0);
+
+    let thread_id1 = handle1.await.unwrap();
+    let thread_id2 = handle2.await.unwrap();
+
+    assert_ne!(thread_id1, thread_id2);
+}

tokio-util/tests/sync_cancellation_token.rs

@@ -1,7 +1,7 @@
 #![warn(rust_2018_idioms)]
 
 use tokio::pin;
-use tokio_util::sync::CancellationToken;
+use tokio_util::sync::{CancellationToken, WaitForCancellationFuture};
 
 use core::future::Future;
 use core::task::{Context, Poll};
@@ -77,6 +77,46 @@ fn cancel_child_token_through_parent() {
     );
 }
 
+#[test]
+fn cancel_grandchild_token_through_parent_if_child_was_dropped() {
+    let (waker, wake_counter) = new_count_waker();
+    let token = CancellationToken::new();
+
+    let intermediate_token = token.child_token();
+    let child_token = intermediate_token.child_token();
+    drop(intermediate_token);
+    assert!(!child_token.is_cancelled());
+
+    let child_fut = child_token.cancelled();
+    pin!(child_fut);
+    let parent_fut = token.cancelled();
+    pin!(parent_fut);
+
+    assert_eq!(
+        Poll::Pending,
+        child_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    token.cancel();
+    assert_eq!(wake_counter, 2);
+    assert!(token.is_cancelled());
+    assert!(child_token.is_cancelled());
+
+    assert_eq!(
+        Poll::Ready(()),
+        child_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+}
+
 #[test]
 fn cancel_child_token_without_parent() {
     let (waker, wake_counter) = new_count_waker();
@@ -206,6 +246,134 @@ fn drop_multiple_child_tokens() {
     }
 }
 
+#[test]
+fn cancel_only_all_descendants() {
+    // ARRANGE
+    let (waker, wake_counter) = new_count_waker();
+
+    let parent_token = CancellationToken::new();
+    let token = parent_token.child_token();
+    let sibling_token = parent_token.child_token();
+    let child1_token = token.child_token();
+    let child2_token = token.child_token();
+    let grandchild_token = child1_token.child_token();
+    let grandchild2_token = child1_token.child_token();
+    let great_grandchild_token = grandchild_token.child_token();
+
+    assert!(!parent_token.is_cancelled());
+    assert!(!token.is_cancelled());
+    assert!(!sibling_token.is_cancelled());
+    assert!(!child1_token.is_cancelled());
+    assert!(!child2_token.is_cancelled());
+    assert!(!grandchild_token.is_cancelled());
+    assert!(!grandchild2_token.is_cancelled());
+    assert!(!great_grandchild_token.is_cancelled());
+
+    let parent_fut = parent_token.cancelled();
+    let fut = token.cancelled();
+    let sibling_fut = sibling_token.cancelled();
+    let child1_fut = child1_token.cancelled();
+    let child2_fut = child2_token.cancelled();
+    let grandchild_fut = grandchild_token.cancelled();
+    let grandchild2_fut = grandchild2_token.cancelled();
+    let great_grandchild_fut = great_grandchild_token.cancelled();
+
+    pin!(parent_fut);
+    pin!(fut);
+    pin!(sibling_fut);
+    pin!(child1_fut);
+    pin!(child2_fut);
+    pin!(grandchild_fut);
+    pin!(grandchild2_fut);
+    pin!(great_grandchild_fut);
+
+    assert_eq!(
+        Poll::Pending,
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        sibling_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        child1_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        child2_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        grandchild2_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        great_grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    // ACT
+    token.cancel();
+
+    // ASSERT
+    assert_eq!(wake_counter, 6);
+    assert!(!parent_token.is_cancelled());
+    assert!(token.is_cancelled());
+    assert!(!sibling_token.is_cancelled());
+    assert!(child1_token.is_cancelled());
+    assert!(child2_token.is_cancelled());
+    assert!(grandchild_token.is_cancelled());
+    assert!(grandchild2_token.is_cancelled());
+    assert!(great_grandchild_token.is_cancelled());
+
+    assert_eq!(
+        Poll::Ready(()),
+        fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        child1_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        child2_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        grandchild2_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        great_grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 6);
+}
+
 #[test]
 fn drop_parent_before_child_tokens() {
     let token = CancellationToken::new();
@@ -218,3 +386,15 @@ fn drop_parent_before_child_tokens() {
     drop(child1);
     drop(child2);
 }
+
+#[test]
+fn derives_send_sync() {
+    fn assert_send<T: Send>() {}
+    fn assert_sync<T: Sync>() {}
+
+    assert_send::<CancellationToken>();
+    assert_sync::<CancellationToken>();
+
+    assert_send::<WaitForCancellationFuture<'static>>();
+    assert_sync::<WaitForCancellationFuture<'static>>();
+}

tokio-util/tests/task_join_map.rs

@@ -24,7 +24,7 @@ async fn test_with_sleep() {
     map.detach_all();
     assert_eq!(map.len(), 0);
 
-    assert!(matches!(map.join_one().await, None));
+    assert!(matches!(map.join_next().await, None));
 
     for i in 0..10 {
         map.spawn(i, async move {
@@ -35,7 +35,7 @@ async fn test_with_sleep() {
     }
 
     let mut seen = [false; 10];
-    while let Some((k, res)) = map.join_one().await {
+    while let Some((k, res)) = map.join_next().await {
         seen[k] = true;
         assert_eq!(res.expect("task should have completed successfully"), k);
     }
@@ -43,7 +43,7 @@ async fn test_with_sleep() {
     for was_seen in &seen {
         assert!(was_seen);
     }
-    assert!(matches!(map.join_one().await, None));
+    assert!(matches!(map.join_next().await, None));
 
     // Do it again.
     for i in 0..10 {
@@ -54,7 +54,7 @@ async fn test_with_sleep() {
     }
 
     let mut seen = [false; 10];
-    while let Some((k, res)) = map.join_one().await {
+    while let Some((k, res)) = map.join_next().await {
         seen[k] = true;
         assert_eq!(res.expect("task should have completed successfully"), k);
     }
@@ -62,7 +62,7 @@ async fn test_with_sleep() {
     for was_seen in &seen {
         assert!(was_seen);
     }
-    assert!(matches!(map.join_one().await, None));
+    assert!(matches!(map.join_next().await, None));
 }
 
 #[tokio::test]
@@ -101,7 +101,7 @@ async fn alternating() {
     assert_eq!(map.len(), 2);
 
     for i in 0..16 {
-        let (_, res) = map.join_one().await.unwrap();
+        let (_, res) = map.join_next().await.unwrap();
         assert!(res.is_ok());
         assert_eq!(map.len(), 1);
         map.spawn(i, async {});
@@ -127,7 +127,7 @@ async fn abort_by_key() {
         }
     }
 
-    while let Some((key, res)) = map.join_one().await {
+    while let Some((key, res)) = map.join_next().await {
         match res {
             Ok(()) => {
                 num_completed += 1;
@@ -161,7 +161,7 @@ async fn abort_by_predicate() {
     // abort odd-numbered tasks.
     map.abort_matching(|key| key % 2 != 0);
 
-    while let Some((key, res)) = map.join_one().await {
+    while let Some((key, res)) = map.join_next().await {
         match res {
             Ok(()) => {
                 num_completed += 1;
@@ -190,12 +190,12 @@ fn runtime_gone() {
         drop(rt);
     }
 
-    let (key, res) = rt().block_on(map.join_one()).unwrap();
+    let (key, res) = rt().block_on(map.join_next()).unwrap();
     assert_eq!(key, "key");
     assert!(res.unwrap_err().is_cancelled());
 }
 
-// This ensures that `join_one` works correctly when the coop budget is
+// This ensures that `join_next` works correctly when the coop budget is
 // exhausted.
 #[tokio::test(flavor = "current_thread")]
 async fn join_map_coop() {
@@ -222,7 +222,7 @@ async fn join_map_coop() {
     let mut count = 0;
     let mut coop_count = 0;
     loop {
-        match map.join_one().now_or_never() {
+        match map.join_next().now_or_never() {
             Some(Some((key, Ok(i)))) => assert_eq!(key, i),
             Some(Some((key, Err(err)))) => panic!("failed[{}]: {}", key, err),
             None => {
@@ -260,7 +260,7 @@ async fn abort_all() {
 
     let mut count = 0;
     let mut seen = [false; 10];
-    while let Some((k, res)) = map.join_one().await {
+    while let Some((k, res)) = map.join_next().await {
         seen[k] = true;
         if let Err(err) = res {
             assert!(err.is_cancelled());

tokio-util/tests/time_delay_queue.rs

@@ -778,6 +778,7 @@ async fn compact_change_deadline() {
     assert!(entry.is_none());
 }
 
+#[cfg_attr(target_os = "wasi", ignore = "FIXME: Does not seem to work with WASI")]
 #[tokio::test(start_paused = true)]
 async fn remove_after_compact() {
     let now = Instant::now();
@@ -794,6 +795,7 @@ async fn remove_after_compact() {
     assert!(panic.is_err());
 }
 
+#[cfg_attr(target_os = "wasi", ignore = "FIXME: Does not seem to work with WASI")]
 #[tokio::test(start_paused = true)]
 async fn remove_after_compact_poll() {
     let now = Instant::now();

tokio-util/tests/udp.rs

@@ -1,4 +1,5 @@
 #![warn(rust_2018_idioms)]
+#![cfg(not(target_os = "wasi"))] // Wasi doesn't support UDP
 
 use tokio::net::UdpSocket;
 use tokio_stream::StreamExt;

tokio/CHANGELOG.md

@@ -1,3 +1,216 @@
+# 1.21.0 (September 2, 2022)
+
+This release is the first release of Tokio to intentionally support WASM. The
+`sync,macros,io-util,rt,time` features are stabilized on WASM. Additionally the
+wasm32-wasi target is given unstable support for the `net` feature.
+
+### Added
+
+- net: add `device` and `bind_device` methods to TCP/UDP sockets ([#4882])
+- net: add `tos` and `set_tos` methods to TCP and UDP sockets ([#4877])
+- net: add security flags to named pipe `ServerOptions` ([#4845])
+- signal: add more windows signal handlers ([#4924])
+- sync: add `mpsc::Sender::max_capacity` method ([#4904])
+- sync: implement Weak version of `mpsc::Sender` ([#4595])
+- task: add `LocalSet::enter` ([#4765])
+- task: stabilize `JoinSet` and `AbortHandle` ([#4920])
+- tokio: add `track_caller` to public APIs ([#4805], [#4848], [#4852])
+- wasm: initial support for `wasm32-wasi` target ([#4716])
+
+### Fixed
+
+- miri: improve miri compatibility by avoiding temporary references in `linked_list::Link` impls ([#4841])
+- signal: don't register write interest on signal pipe ([#4898])
+- sync: add `#[must_use]` to lock guards ([#4886])
+- sync: fix hang when calling `recv` on closed and reopened broadcast channel ([#4867])
+- task: propagate attributes on task-locals ([#4837])
+
+### Changed
+
+- fs: change panic to error in `File::start_seek` ([#4897])
+- io: reduce syscalls in `poll_read` ([#4840])
+- process: use blocking threadpool for child stdio I/O ([#4824])
+- signal: make `SignalKind` methods const ([#4956])
+
+### Internal changes
+
+- rt: extract `basic_scheduler::Config` ([#4935])
+- rt: move I/O driver into `runtime` module ([#4942])
+- rt: rename internal scheduler types ([#4945])
+
+### Documented
+
+- chore: fix typos and grammar ([#4858], [#4894], [#4928])
+- io: fix typo in `AsyncSeekExt::rewind` docs ([#4893])
+- net: add documentation to `try_read()` for zero-length buffers ([#4937])
+- runtime: remove incorrect panic section for `Builder::worker_threads` ([#4849])
+- sync: doc of `watch::Sender::send` improved ([#4959])
+- task: add cancel safety docs to `JoinHandle` ([#4901])
+- task: expand on cancellation of `spawn_blocking` ([#4811])
+- time: clarify that the first tick of `Interval::tick` happens immediately ([#4951])
+
+### Unstable
+
+- rt: add unstable option to disable the LIFO slot ([#4936])
+- task: fix incorrect signature in `Builder::spawn_on` ([#4953])
+- task: make `task::Builder::spawn*` methods fallible ([#4823])
+
+[#4595]: https://github.com/tokio-rs/tokio/pull/4595
+[#4716]: https://github.com/tokio-rs/tokio/pull/4716
+[#4765]: https://github.com/tokio-rs/tokio/pull/4765
+[#4805]: https://github.com/tokio-rs/tokio/pull/4805
+[#4811]: https://github.com/tokio-rs/tokio/pull/4811
+[#4823]: https://github.com/tokio-rs/tokio/pull/4823
+[#4824]: https://github.com/tokio-rs/tokio/pull/4824
+[#4837]: https://github.com/tokio-rs/tokio/pull/4837
+[#4840]: https://github.com/tokio-rs/tokio/pull/4840
+[#4841]: https://github.com/tokio-rs/tokio/pull/4841
+[#4845]: https://github.com/tokio-rs/tokio/pull/4845
+[#4848]: https://github.com/tokio-rs/tokio/pull/4848
+[#4849]: https://github.com/tokio-rs/tokio/pull/4849
+[#4852]: https://github.com/tokio-rs/tokio/pull/4852
+[#4858]: https://github.com/tokio-rs/tokio/pull/4858
+[#4867]: https://github.com/tokio-rs/tokio/pull/4867
+[#4877]: https://github.com/tokio-rs/tokio/pull/4877
+[#4882]: https://github.com/tokio-rs/tokio/pull/4882
+[#4886]: https://github.com/tokio-rs/tokio/pull/4886
+[#4893]: https://github.com/tokio-rs/tokio/pull/4893
+[#4894]: https://github.com/tokio-rs/tokio/pull/4894
+[#4897]: https://github.com/tokio-rs/tokio/pull/4897
+[#4898]: https://github.com/tokio-rs/tokio/pull/4898
+[#4901]: https://github.com/tokio-rs/tokio/pull/4901
+[#4904]: https://github.com/tokio-rs/tokio/pull/4904
+[#4920]: https://github.com/tokio-rs/tokio/pull/4920
+[#4924]: https://github.com/tokio-rs/tokio/pull/4924
+[#4928]: https://github.com/tokio-rs/tokio/pull/4928
+[#4935]: https://github.com/tokio-rs/tokio/pull/4935
+[#4936]: https://github.com/tokio-rs/tokio/pull/4936
+[#4937]: https://github.com/tokio-rs/tokio/pull/4937
+[#4942]: https://github.com/tokio-rs/tokio/pull/4942
+[#4945]: https://github.com/tokio-rs/tokio/pull/4945
+[#4951]: https://github.com/tokio-rs/tokio/pull/4951
+[#4953]: https://github.com/tokio-rs/tokio/pull/4953
+[#4956]: https://github.com/tokio-rs/tokio/pull/4956
+[#4959]: https://github.com/tokio-rs/tokio/pull/4959
+
+# 1.20.1 (July 25, 2022)
+
+### Fixed
+
+- chore: fix version detection in build script ([#4860])
+
+[#4860]: https://github.com/tokio-rs/tokio/pull/4860
+
+# 1.20.0 (July 12, 2022)
+
+### Added
+- tokio: add `track_caller` to public APIs ([#4772], [#4791], [#4793], [#4806], [#4808])
+- sync: Add `has_changed` method to `watch::Ref` ([#4758])
+
+### Changed
+
+- time: remove `src/time/driver/wheel/stack.rs` ([#4766])
+- rt: clean up arguments passed to basic scheduler ([#4767])
+- net: be more specific about winapi features ([#4764])
+- tokio: use const initialized thread locals where possible ([#4677])
+- task: various small improvements to LocalKey ([#4795])
+
+### Documented
+
+- fs: warn about performance pitfall ([#4762])
+- chore: fix spelling ([#4769])
+- sync: document spurious failures in oneshot ([#4777])
+- sync: add warning for watch in non-Send futures ([#4741])
+- chore: fix typo ([#4798])
+
+### Unstable
+
+- joinset: rename `join_one` to `join_next` ([#4755])
+- rt: unhandled panic config for current thread rt ([#4770])
+
+[#4677]: https://github.com/tokio-rs/tokio/pull/4677
+[#4741]: https://github.com/tokio-rs/tokio/pull/4741
+[#4755]: https://github.com/tokio-rs/tokio/pull/4755
+[#4758]: https://github.com/tokio-rs/tokio/pull/4758
+[#4762]: https://github.com/tokio-rs/tokio/pull/4762
+[#4764]: https://github.com/tokio-rs/tokio/pull/4764
+[#4766]: https://github.com/tokio-rs/tokio/pull/4766
+[#4767]: https://github.com/tokio-rs/tokio/pull/4767
+[#4769]: https://github.com/tokio-rs/tokio/pull/4769
+[#4770]: https://github.com/tokio-rs/tokio/pull/4770
+[#4772]: https://github.com/tokio-rs/tokio/pull/4772
+[#4777]: https://github.com/tokio-rs/tokio/pull/4777
+[#4791]: https://github.com/tokio-rs/tokio/pull/4791
+[#4793]: https://github.com/tokio-rs/tokio/pull/4793
+[#4795]: https://github.com/tokio-rs/tokio/pull/4795
+[#4798]: https://github.com/tokio-rs/tokio/pull/4798
+[#4806]: https://github.com/tokio-rs/tokio/pull/4806
+[#4808]: https://github.com/tokio-rs/tokio/pull/4808
+
+# 1.19.2 (June 6, 2022)
+
+This release fixes another bug in `Notified::enable`. ([#4751])
+
+[#4751]: https://github.com/tokio-rs/tokio/pull/4751
+
+# 1.19.1 (June 5, 2022)
+
+This release fixes a bug in `Notified::enable`. ([#4747])
+
+[#4747]: https://github.com/tokio-rs/tokio/pull/4747
+
+# 1.19.0 (June 3, 2022)
+
+### Added
+
+- runtime: add `is_finished` method for `JoinHandle` and `AbortHandle` ([#4709])
+- runtime: make global queue and event polling intervals configurable ([#4671])
+- sync: add `Notified::enable` ([#4705])
+- sync: add `watch::Sender::send_if_modified` ([#4591])
+- sync: add resubscribe method to broadcast::Receiver ([#4607])
+- net: add `take_error` to `TcpSocket` and `TcpStream` ([#4739])
+
+### Changed
+
+- io: refactor out usage of Weak in the io handle ([#4656])
+
+### Fixed
+
+- macros: avoid starvation in `join!` and `try_join!` ([#4624])
+
+### Documented
+
+- runtime: clarify semantics of tasks outliving `block_on` ([#4729])
+- time: fix example for `MissedTickBehavior::Burst` ([#4713])
+
+### Unstable
+
+- metrics: correctly update atomics in `IoDriverMetrics` ([#4725])
+- metrics: fix compilation with unstable, process, and rt, but without net ([#4682])
+- task: add `#[track_caller]` to `JoinSet`/`JoinMap` ([#4697])
+- task: add `Builder::{spawn_on, spawn_local_on, spawn_blocking_on}` ([#4683])
+- task: add `consume_budget` for cooperative scheduling ([#4498])
+- task: add `join_set::Builder` for configuring `JoinSet` tasks ([#4687])
+- task: update return value of `JoinSet::join_one` ([#4726])
+
+[#4498]: https://github.com/tokio-rs/tokio/pull/4498
+[#4591]: https://github.com/tokio-rs/tokio/pull/4591
+[#4607]: https://github.com/tokio-rs/tokio/pull/4607
+[#4624]: https://github.com/tokio-rs/tokio/pull/4624
+[#4656]: https://github.com/tokio-rs/tokio/pull/4656
+[#4671]: https://github.com/tokio-rs/tokio/pull/4671
+[#4682]: https://github.com/tokio-rs/tokio/pull/4682
+[#4683]: https://github.com/tokio-rs/tokio/pull/4683
+[#4687]: https://github.com/tokio-rs/tokio/pull/4687
+[#4697]: https://github.com/tokio-rs/tokio/pull/4697
+[#4705]: https://github.com/tokio-rs/tokio/pull/4705
+[#4709]: https://github.com/tokio-rs/tokio/pull/4709
+[#4713]: https://github.com/tokio-rs/tokio/pull/4713
+[#4725]: https://github.com/tokio-rs/tokio/pull/4725
+[#4726]: https://github.com/tokio-rs/tokio/pull/4726
+[#4729]: https://github.com/tokio-rs/tokio/pull/4729
+[#4739]: https://github.com/tokio-rs/tokio/pull/4739
+
 # 1.18.2 (May 5, 2022)
 
 Add missing features for the `winapi` dependency. ([#4663])
@@ -27,8 +240,6 @@ task), as well as a number of bugfixes.
 
 ### Added
 
-- macros: support setting a custom crate name for `#[tokio::main]` and
-  `#[tokio::test]` ([#4613])
 - net: add `UdpSocket::peer_addr` ([#4611])
 - net: add `try_read_buf` method for named pipes ([#4626])
 - signal: add `SignalKind` `Hash`/`Eq` impls and `c_int` conversion ([#4540])
@@ -1173,7 +1384,7 @@ Biggest changes are:
 - Feature flags are simplified
   - `rt-core` and `rt-util` are combined to `rt`
   - `rt-threaded` is renamed to `rt-multi-thread` to match builder API
-  - `tcp`, `udp`, `uds`, `dns` are combied to `net`.
+  - `tcp`, `udp`, `uds`, `dns` are combined to `net`.
   - `parking_lot` is included with `full`
 
 ### Changes
@@ -1671,7 +1882,7 @@ Biggest changes are:
 - `net::lookup_host` maps a `T: ToSocketAddrs` to a stream of `SocketAddrs` ([#1870]).
 - `process::Child` fields are made public to match `std` ([#2014]).
 - impl `Stream` for `sync::broadcast::Receiver` ([#2012]).
-- `sync::RwLock` provides an asynchonous read-write lock ([#1699]).
+- `sync::RwLock` provides an asynchronous read-write lock ([#1699]).
 - `runtime::Handle::current` returns the handle for the current runtime ([#2040]).
 - `StreamExt::filter` filters stream values according to a predicate ([#2001]).
 - `StreamExt::filter_map` simultaneously filter and map stream values ([#2001]).
@@ -1780,7 +1991,7 @@ Biggest changes are:
 ### Fixes
 
 - calling `spawn_blocking` after runtime shutdown ([#1875]).
-- `LocalSet` drop inifinite loop ([#1892]).
+- `LocalSet` drop infinite loop ([#1892]).
 - `LocalSet` hang under load ([#1905]).
 - improved documentation ([#1865], [#1866], [#1868], [#1874], [#1876], [#1911]).
 

tokio/Cargo.toml

@@ -6,7 +6,7 @@ name = "tokio"
 #   - README.md
 # - Update CHANGELOG.md.
 # - Create "v1.0.x" git tag.
-version = "1.18.2"
+version = "1.21.0"
 edition = "2018"
 rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
@@ -51,8 +51,14 @@ net = [
   "mio/os-poll",
   "mio/os-ext",
   "mio/net",
-  "socket2",
+  "winapi/fileapi",
+  "winapi/handleapi",
   "winapi/namedpipeapi",
+  "winapi/winbase",
+  "winapi/winnt",
+  "winapi/minwindef",
+  "winapi/accctrl",
+  "winapi/aclapi"
 ]
 process = [
   "bytes",
@@ -62,7 +68,12 @@ process = [
   "mio/os-ext",
   "mio/net",
   "signal-hook-registry",
+  "winapi/handleapi",
+  "winapi/minwindef",
+  "winapi/processthreadsapi",
   "winapi/threadpoollegacyapiset",
+  "winapi/winbase",
+  "winapi/winnt",
 ]
 # Includes basic task execution capabilities
 rt = ["once_cell"]
@@ -78,6 +89,8 @@ signal = [
   "mio/os-ext",
   "signal-hook-registry",
   "winapi/consoleapi",
+  "winapi/wincon",
+  "winapi/minwindef",
 ]
 sync = []
 test-util = ["rt", "sync", "time"]
@@ -88,6 +101,9 @@ time = []
 # a few releases.
 stats = []
 
+[build-dependencies]
+autocfg = "1.1"
+
 [dependencies]
 tokio-macros = { version = "1.7.0", path = "../tokio-macros", optional = true }
 
@@ -97,11 +113,13 @@ pin-project-lite = "0.2.0"
 bytes = { version = "1.0.0", optional = true }
 once_cell = { version = "1.5.2", optional = true }
 memchr = { version = "2.2", optional = true }
-mio = { version = "0.8.1", optional = true }
-socket2 = { version = "0.4.4", optional = true, features = [ "all" ] }
+mio = { version = "0.8.4", optional = true }
 num_cpus = { version = "1.8.0", optional = true }
 parking_lot = { version = "0.12.0", optional = true }
 
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dependencies]
+socket2 = { version = "0.4.4", features = [ "all" ] }
+
 # Currently unstable. The API exposed by these features may be broken at any time.
 # Requires `--cfg tokio_unstable` to enable.
 [target.'cfg(tokio_unstable)'.dependencies]
@@ -118,7 +136,7 @@ nix = { version = "0.24", default-features = false, features = ["fs", "socket"]
 [target.'cfg(windows)'.dependencies.winapi]
 version = "0.3.8"
 default-features = false
-features = ["std", "winsock2", "mswsock", "handleapi", "ws2ipdef", "ws2tcpip"]
+features = ["std"]
 optional = true
 
 [target.'cfg(windows)'.dev-dependencies.ntapi]
@@ -128,16 +146,18 @@ version = "0.3.6"
 tokio-test = { version = "0.4.0", path = "../tokio-test" }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 futures = { version = "0.3.0", features = ["async-await"] }
-mockall = "0.10.2"
+mockall = "0.11.1"
 tempfile = "3.1.0"
 async-stream = "0.3"
 
-[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dev-dependencies]
 proptest = "1"
-rand = "0.8.0"
 socket2 = "0.4"
 
-[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+[target.'cfg(not(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown")))'.dev-dependencies]
+rand = "0.8.0"
+
+[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), not(target_os = "wasi")))'.dev-dependencies]
 wasm-bindgen-test = "0.3.0"
 
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]

tokio/README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.18.2", features = ["full"] }
+tokio = { version = "1.21.0", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -161,8 +161,30 @@ several other libraries, including:
 [`mio`]: https://github.com/tokio-rs/mio
 [`bytes`]: https://github.com/tokio-rs/bytes
 
+## Changelog
+
+The Tokio repository contains multiple crates. Each crate has its own changelog.
+
+ * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md)
+ * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md)
+ * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md)
+ * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md)
+ * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md)
+
 ## Supported Rust Versions
 
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- CONTRIBUTING.md
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
 Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
 least** 6 months. When increasing the MSRV, the new Rust version must have been
 released at least six months ago. The current MSRV is 1.49.0.
@@ -180,18 +202,18 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.8.x` - LTS release until February 2022.
- * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until June 2023
+ * `1.20.x` - LTS release until September 2023.
 
-Each LTS release will continue to receive backported fixes for at least half a
-year. If you wish to use a fixed minor release in your project, we recommend
-that you use an LTS release.
+Each LTS release will continue to receive backported fixes for at least a year.
+If you wish to use a fixed minor release in your project, we recommend that you
+use an LTS release.
 
 To use a fixed minor version, you can specify the version with a tilde. For
-example, to specify that you wish to use the newest `1.8.x` patch release, you
+example, to specify that you wish to use the newest `1.18.x` patch release, you
 can use the following dependency specification:
 ```text
-tokio = { version = "~1.8", features = [...] }
+tokio = { version = "~1.18", features = [...] }
 ```
 
 ## License

tokio/build.rs

@@ -0,0 +1,101 @@
+use autocfg::AutoCfg;
+
+const CONST_THREAD_LOCAL_PROBE: &str = r#"
+{
+    thread_local! {
+        static MY_PROBE: usize = const { 10 };
+    }
+
+    MY_PROBE.with(|val| *val)
+}
+"#;
+
+const ADDR_OF_PROBE: &str = r#"
+{
+    let my_var = 10;
+    ::std::ptr::addr_of!(my_var)
+}
+"#;
+
+fn main() {
+    let mut enable_const_thread_local = false;
+    let mut enable_addr_of = false;
+
+    match AutoCfg::new() {
+        Ok(ac) => {
+            // These checks prefer to call only `probe_rustc_version` if that is
+            // enough to determine whether the feature is supported. This is
+            // because the `probe_expression` call involves a call to rustc,
+            // which the `probe_rustc_version` call avoids.
+
+            // Const-initialized thread locals were stabilized in 1.59.
+            if ac.probe_rustc_version(1, 60) {
+                enable_const_thread_local = true;
+            } else if ac.probe_rustc_version(1, 59) {
+                // This compiler claims to be 1.59, but there are some nightly
+                // compilers that claim to be 1.59 without supporting the
+                // feature. Explicitly probe to check if code using them
+                // compiles.
+                //
+                // The oldest nightly that supports the feature is 2021-12-06.
+                if ac.probe_expression(CONST_THREAD_LOCAL_PROBE) {
+                    enable_const_thread_local = true;
+                }
+            }
+
+            // The `addr_of` and `addr_of_mut` macros were stabilized in 1.51.
+            if ac.probe_rustc_version(1, 52) {
+                enable_addr_of = true;
+            } else if ac.probe_rustc_version(1, 51) {
+                // This compiler claims to be 1.51, but there are some nightly
+                // compilers that claim to be 1.51 without supporting the
+                // feature. Explicitly probe to check if code using them
+                // compiles.
+                //
+                // The oldest nightly that supports the feature is 2021-01-31.
+                if ac.probe_expression(ADDR_OF_PROBE) {
+                    enable_addr_of = true;
+                }
+            }
+        }
+
+        Err(e) => {
+            // If we couldn't detect the compiler version and features, just
+            // print a warning. This isn't a fatal error: we can still build
+            // Tokio, we just can't enable cfgs automatically.
+            println!(
+                "cargo:warning=tokio: failed to detect compiler features: {}",
+                e
+            );
+        }
+    }
+
+    if !enable_const_thread_local {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_const_thread_local"
+        autocfg::emit("tokio_no_const_thread_local")
+    }
+
+    if !enable_addr_of {
+        // To disable this feature on compilers that support it, you can
+        // explicitly pass this flag with the following environment variable:
+        //
+        // RUSTFLAGS="--cfg tokio_no_addr_of"
+        autocfg::emit("tokio_no_addr_of")
+    }
+
+    let target = ::std::env::var("TARGET").unwrap_or_default();
+
+    // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54.
+    // Note that these cfgs are unavailable in `Cargo.toml`.
+    if target.starts_with("wasm") {
+        autocfg::emit("tokio_wasm");
+        if target.contains("wasi") {
+            autocfg::emit("tokio_wasi");
+        } else {
+            autocfg::emit("tokio_wasm_not_wasi");
+        }
+    }
+}

tokio/docs/reactor-refactor.md

@@ -188,9 +188,9 @@ readiness, the driver's tick is packed into the atomic `usize`.
 The `ScheduledIo` readiness `AtomicUsize` is structured as:
 
 ```
-| reserved | generation |  driver tick | readinesss |
-|----------+------------+--------------+------------|
-|   1 bit  |   7 bits   +    8 bits    +   16 bits  |
+| reserved | generation |  driver tick | readiness |
+|----------+------------+--------------+-----------|
+|   1 bit  |   7 bits   +    8 bits    +  16 bits  |
 ```
 
 The `reserved` and `generation` components exist today.

tokio/external-types.toml

@@ -0,0 +1,16 @@
+# This config file is for the `cargo-check-external-types` tool that is run in CI.
+
+# The following are types that are allowed to be exposed in Tokio's public API.
+# The standard library is allowed by default.
+allowed_external_types = [
+   "bytes::buf::buf_impl::Buf",
+   "bytes::buf::buf_mut::BufMut",
+
+   "tokio_macros::*",
+
+   # TODO(https://github.com/tokio-rs/tokio/issues/4916): Remove the libc types
+   "libc::unix::gid_t",
+   "libc::unix::pid_t",
+   "libc::unix::uid_t",
+]
+

tokio/src/coop.rs

@@ -32,7 +32,7 @@
 use std::cell::Cell;
 
 thread_local! {
-    static CURRENT: Cell<Budget> = Cell::new(Budget::unconstrained());
+    static CURRENT: Cell<Budget> = const { Cell::new(Budget::unconstrained()) };
 }
 
 /// Opaque type tracking the amount of "work" a task may still do before
@@ -207,7 +207,7 @@ cfg_coop! {
 mod test {
     use super::*;
 
-    #[cfg(target_arch = "wasm32")]
+    #[cfg(tokio_wasm_not_wasi)]
     use wasm_bindgen_test::wasm_bindgen_test as test;
 
     fn get() -> Budget {
@@ -215,7 +215,7 @@ mod test {
     }
 
     #[test]
-    fn bugeting() {
+    fn budgeting() {
         use futures::future::poll_fn;
         use tokio_test::*;
 

tokio/src/fs/file.rs

@@ -565,29 +565,30 @@ impl AsyncSeek for File {
         let me = self.get_mut();
         let inner = me.inner.get_mut();
 
-        loop {
-            match inner.state {
-                Busy(_) => panic!("must wait for poll_complete before calling start_seek"),
-                Idle(ref mut buf_cell) => {
-                    let mut buf = buf_cell.take().unwrap();
+        match inner.state {
+            Busy(_) => Err(io::Error::new(
+                io::ErrorKind::Other,
+                "other file operation is pending, call poll_complete before start_seek",
+            )),
+            Idle(ref mut buf_cell) => {
+                let mut buf = buf_cell.take().unwrap();
 
-                    // Factor in any unread data from the buf
-                    if !buf.is_empty() {
-                        let n = buf.discard_read();
+                // Factor in any unread data from the buf
+                if !buf.is_empty() {
+                    let n = buf.discard_read();
 
-                        if let SeekFrom::Current(ref mut offset) = pos {
-                            *offset += n;
-                        }
+                    if let SeekFrom::Current(ref mut offset) = pos {
+                        *offset += n;
                     }
-
-                    let std = me.std.clone();
-
-                    inner.state = Busy(spawn_blocking(move || {
-                        let res = (&*std).seek(pos);
-                        (Operation::Seek(res), buf)
-                    }));
-                    return Ok(());
                 }
+
+                let std = me.std.clone();
+
+                inner.state = Busy(spawn_blocking(move || {
+                    let res = (&*std).seek(pos);
+                    (Operation::Seek(res), buf)
+                }));
+                Ok(())
             }
         }
     }

tokio/src/fs/file/tests.rs

@@ -955,3 +955,24 @@ fn partial_read_set_len_ok() {
     assert_eq!(n, FOO.len());
     assert_eq!(&buf[..n], FOO);
 }
+
+#[test]
+fn busy_file_seek_error() {
+    let mut file = MockFile::default();
+    let mut seq = Sequence::new();
+    file.expect_inner_write()
+        .once()
+        .in_sequence(&mut seq)
+        .returning(|_| Err(io::ErrorKind::Other.into()));
+
+    let mut file = crate::io::BufReader::new(File::from_std(file));
+    {
+        let mut t = task::spawn(file.write(HELLO));
+        assert_ready_ok!(t.poll());
+    }
+
+    pool::run_one();
+
+    let mut t = task::spawn(file.seek(SeekFrom::Start(0)));
+    assert_ready_err!(t.poll());
+}

tokio/src/fs/mod.rs

@@ -22,6 +22,24 @@
 //! `std::io::ErrorKind::WouldBlock` if a *worker* thread can not be converted
 //! to a *backup* thread immediately.
 //!
+//! **Warning**: These adapters may create a large number of temporary tasks,
+//! especially when reading large files. When performing a lot of operations
+//! in one batch, it may be significantly faster to use [`spawn_blocking`]
+//! directly:
+//!
+//! ```
+//! use tokio::fs::File;
+//! use std::io::{BufReader, BufRead};
+//! async fn count_lines(file: File) -> Result<usize, std::io::Error> {
+//!     let file = file.into_std().await;
+//!     tokio::task::spawn_blocking(move || {
+//!         let line_count = BufReader::new(file).lines().count();
+//!         Ok(line_count)
+//!     }).await?
+//! }
+//! ```
+//!
+//! [`spawn_blocking`]: fn@crate::task::spawn_blocking
 //! [`AsyncRead`]: trait@crate::io::AsyncRead
 
 mod canonicalize;

tokio/src/fs/read_dir.rs

@@ -34,7 +34,7 @@ pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
     Ok(ReadDir(State::Idle(Some(std))))
 }
 
-/// Reads the the entries in a directory.
+/// Reads the entries in a directory.
 ///
 /// This struct is returned from the [`read_dir`] function of this module and
 /// will yield instances of [`DirEntry`]. Through a [`DirEntry`] information

tokio/src/future/block_on.rs

@@ -1,6 +1,7 @@
 use std::future::Future;
 
 cfg_rt! {
+    #[track_caller]
     pub(crate) fn block_on<F: Future>(f: F) -> F::Output {
         let mut e = crate::runtime::enter::enter(false);
         e.block_on(f).unwrap()
@@ -8,6 +9,7 @@ cfg_rt! {
 }
 
 cfg_not_rt! {
+    #[track_caller]
     pub(crate) fn block_on<F: Future>(f: F) -> F::Output {
         let mut park = crate::park::thread::CachedParkThread::new();
         park.block_on(f).unwrap()

tokio/src/future/poll_fn.rs

@@ -35,6 +35,6 @@ where
     type Output = T;
 
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<T> {
-        (&mut self.f)(cx)
+        (self.f)(cx)
     }
 }

tokio/src/io/async_fd.rs

@@ -1,4 +1,5 @@
-use crate::io::driver::{Handle, Interest, ReadyEvent, Registration};
+use crate::io::Interest;
+use crate::runtime::io::{Handle, ReadyEvent, Registration};
 
 use mio::unix::SourceFd;
 use std::io;
@@ -167,12 +168,18 @@ pub struct AsyncFdReadyMutGuard<'a, T: AsRawFd> {
 const ALL_INTEREST: Interest = Interest::READABLE.add(Interest::WRITABLE);
 
 impl<T: AsRawFd> AsyncFd<T> {
-    #[inline]
     /// Creates an AsyncFd backed by (and taking ownership of) an object
     /// implementing [`AsRawFd`]. The backing file descriptor is cached at the
     /// time of creation.
     ///
     /// This method must be called in the context of a tokio runtime.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if there is no current reactor set, or if the `rt`
+    /// feature flag is not enabled.
+    #[inline]
+    #[track_caller]
     pub fn new(inner: T) -> io::Result<Self>
     where
         T: AsRawFd,
@@ -180,9 +187,15 @@ impl<T: AsRawFd> AsyncFd<T> {
         Self::with_interest(inner, ALL_INTEREST)
     }
 
-    #[inline]
     /// Creates new instance as `new` with additional ability to customize interest,
     /// allowing to specify whether file descriptor will be polled for read, write or both.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if there is no current reactor set, or if the `rt`
+    /// feature flag is not enabled.
+    #[inline]
+    #[track_caller]
     pub fn with_interest(inner: T, interest: Interest) -> io::Result<Self>
     where
         T: AsRawFd,

tokio/src/io/blocking.rs

@@ -34,8 +34,9 @@ enum State<T> {
     Busy(sys::Blocking<(io::Result<usize>, Buf, T)>),
 }
 
-cfg_io_std! {
+cfg_io_blocking! {
     impl<T> Blocking<T> {
+        #[cfg_attr(feature = "fs", allow(dead_code))]
         pub(crate) fn new(inner: T) -> Blocking<T> {
             Blocking {
                 inner: Some(inner),

tokio/src/io/bsd/poll_aio.rs

@@ -1,6 +1,7 @@
 //! Use POSIX AIO futures with Tokio.
 
-use crate::io::driver::{Handle, Interest, ReadyEvent, Registration};
+use crate::io::interest::Interest;
+use crate::runtime::io::{Handle, ReadyEvent, Registration};
 use mio::event::Source;
 use mio::Registry;
 use mio::Token;

tokio/src/io/interest.rs

@@ -1,6 +1,6 @@
 #![cfg_attr(not(feature = "net"), allow(dead_code, unreachable_pub))]
 
-use crate::io::driver::Ready;
+use crate::io::ready::Ready;
 
 use std::fmt;
 use std::ops;
@@ -100,7 +100,7 @@ impl Interest {
         self.0
     }
 
-    pub(super) fn mask(self) -> Ready {
+    pub(crate) fn mask(self) -> Ready {
         match self {
             Interest::READABLE => Ready::READABLE | Ready::READ_CLOSED,
             Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED,

tokio/src/io/mod.rs

@@ -1,5 +1,3 @@
-#![cfg_attr(loom, allow(dead_code, unreachable_pub))]
-
 //! Traits, helpers, and type definitions for asynchronous I/O functionality.
 //!
 //! This module is the asynchronous version of `std::io`. Primarily, it
@@ -205,15 +203,19 @@ pub use self::read_buf::ReadBuf;
 pub use std::io::{Error, ErrorKind, Result, SeekFrom};
 
 cfg_io_driver_impl! {
-    pub(crate) mod driver;
+    pub(crate) mod interest;
+    pub(crate) mod ready;
 
     cfg_net! {
-        pub use driver::{Interest, Ready};
+        pub use interest::Interest;
+        pub use ready::Ready;
     }
 
+    #[cfg_attr(tokio_wasi, allow(unused_imports))]
     mod poll_evented;
 
     #[cfg(not(loom))]
+    #[cfg_attr(tokio_wasi, allow(unused_imports))]
     pub(crate) use poll_evented::PollEvented;
 }
 

tokio/src/io/poll_evented.rs

@@ -1,4 +1,5 @@
-use crate::io::driver::{Handle, Interest, Registration};
+use crate::io::interest::Interest;
+use crate::runtime::io::{Handle, Registration};
 
 use mio::event::Source;
 use std::fmt;
@@ -11,7 +12,7 @@ cfg_io_driver! {
     /// [`std::io::Write`] traits with the reactor that drives it.
     ///
     /// `PollEvented` uses [`Registration`] internally to take a type that
-    /// implements [`mio::event::Source`] as well as [`std::io::Read`] and or
+    /// implements [`mio::event::Source`] as well as [`std::io::Read`] and/or
     /// [`std::io::Write`] and associate it with a reactor that will drive it.
     ///
     /// Once the [`mio::event::Source`] type is wrapped by `PollEvented`, it can be
@@ -41,12 +42,12 @@ cfg_io_driver! {
     /// [`poll_read_ready`] again will also indicate read readiness.
     ///
     /// When the operation is attempted and is unable to succeed due to the I/O
-    /// resource not being ready, the caller must call `clear_readiness`.
+    /// resource not being ready, the caller must call [`clear_readiness`].
     /// This clears the readiness state until a new readiness event is received.
     ///
     /// This allows the caller to implement additional functions. For example,
     /// [`TcpListener`] implements poll_accept by using [`poll_read_ready`] and
-    /// `clear_read_ready`.
+    /// [`clear_readiness`].
     ///
     /// ## Platform-specific events
     ///
@@ -57,6 +58,7 @@ cfg_io_driver! {
     /// [`AsyncRead`]: crate::io::AsyncRead
     /// [`AsyncWrite`]: crate::io::AsyncWrite
     /// [`TcpListener`]: crate::net::TcpListener
+    /// [`clear_readiness`]: Registration::clear_readiness
     /// [`poll_read_ready`]: Registration::poll_read_ready
     /// [`poll_write_ready`]: Registration::poll_write_ready
     pub(crate) struct PollEvented<E: Source> {
@@ -77,6 +79,7 @@ impl<E: Source> PollEvented<E> {
     /// The runtime is usually set implicitly when this function is called
     /// from a future driven by a tokio runtime, otherwise runtime can be set
     /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+    #[track_caller]
     #[cfg_attr(feature = "signal", allow(unused))]
     pub(crate) fn new(io: E) -> io::Result<Self> {
         PollEvented::new_with_interest(io, Interest::READABLE | Interest::WRITABLE)
@@ -97,6 +100,7 @@ impl<E: Source> PollEvented<E> {
     /// a future driven by a tokio runtime, otherwise runtime can be set
     /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter)
     /// function.
+    #[track_caller]
     #[cfg_attr(feature = "signal", allow(unused))]
     pub(crate) fn new_with_interest(io: E, interest: Interest) -> io::Result<Self> {
         Self::new_with_interest_and_handle(io, interest, Handle::current())
@@ -134,7 +138,7 @@ impl<E: Source> PollEvented<E> {
 }
 
 feature! {
-    #![any(feature = "net", feature = "process")]
+    #![any(feature = "net", all(unix, feature = "process"))]
 
     use crate::io::ReadBuf;
     use std::task::{Context, Poll};
@@ -151,16 +155,32 @@ feature! {
         {
             use std::io::Read;
 
-            let n = ready!(self.registration.poll_read_io(cx, || {
-                let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]);
-                self.io.as_ref().unwrap().read(b)
-            }))?;
+            loop {
+                let evt = ready!(self.registration.poll_read_ready(cx))?;
 
-            // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the
-            // buffer.
-            buf.assume_init(n);
-            buf.advance(n);
-            Poll::Ready(Ok(()))
+                let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]);
+                let len = b.len();
+
+                match self.io.as_ref().unwrap().read(b) {
+                    Ok(n) => {
+                        // if we read a partially full buffer, this is sufficient on unix to show
+                        // that the socket buffer has been drained
+                        if n > 0 && (!cfg!(windows) && n < len) {
+                            self.registration.clear_readiness(evt);
+                        }
+
+                        // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the
+                        // buffer.
+                        buf.assume_init(n);
+                        buf.advance(n);
+                        return Poll::Ready(Ok(()));
+                    },
+                    Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+                        self.registration.clear_readiness(evt);
+                    }
+                    Err(e) => return Poll::Ready(Err(e)),
+                }
+            }
         }
 
         pub(crate) fn poll_write<'a>(&'a self, cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>>
@@ -168,7 +188,26 @@ feature! {
             &'a E: io::Write + 'a,
         {
             use std::io::Write;
-            self.registration.poll_write_io(cx, || self.io.as_ref().unwrap().write(buf))
+
+            loop {
+                let evt = ready!(self.registration.poll_write_ready(cx))?;
+
+                match self.io.as_ref().unwrap().write(buf) {
+                    Ok(n) => {
+                        // if we write only part of our buffer, this is sufficient on unix to show
+                        // that the socket buffer is full
+                        if n > 0 && (!cfg!(windows) && n < buf.len()) {
+                            self.registration.clear_readiness(evt);
+                        }
+
+                        return Poll::Ready(Ok(n));
+                    },
+                    Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+                        self.registration.clear_readiness(evt);
+                    }
+                    Err(e) => return Poll::Ready(Err(e)),
+                }
+            }
         }
 
         #[cfg(feature = "net")]

tokio/src/io/read_buf.rs

@@ -152,6 +152,7 @@ impl<'a> ReadBuf<'a> {
     ///
     /// Panics if `self.remaining()` is less than `n`.
     #[inline]
+    #[track_caller]
     pub fn initialize_unfilled_to(&mut self, n: usize) -> &mut [u8] {
         assert!(self.remaining() >= n, "n overflows remaining");
 
@@ -195,6 +196,7 @@ impl<'a> ReadBuf<'a> {
     ///
     /// Panics if the filled region of the buffer would become larger than the initialized region.
     #[inline]
+    #[track_caller]
     pub fn advance(&mut self, n: usize) {
         let new = self.filled.checked_add(n).expect("filled overflow");
         self.set_filled(new);
@@ -211,6 +213,7 @@ impl<'a> ReadBuf<'a> {
     ///
     /// Panics if the filled region of the buffer would become larger than the initialized region.
     #[inline]
+    #[track_caller]
     pub fn set_filled(&mut self, n: usize) {
         assert!(
             n <= self.initialized,
@@ -241,6 +244,7 @@ impl<'a> ReadBuf<'a> {
     ///
     /// Panics if `self.remaining()` is less than `buf.len()`.
     #[inline]
+    #[track_caller]
     pub fn put_slice(&mut self, buf: &[u8]) {
         assert!(
             self.remaining() >= buf.len(),

tokio/src/io/split.rs

@@ -74,6 +74,7 @@ impl<T> ReadHalf<T> {
     /// same `split` operation this method will panic.
     /// This can be checked ahead of time by comparing the stream ID
     /// of the two halves.
+    #[track_caller]
     pub fn unsplit(self, wr: WriteHalf<T>) -> T {
         if self.is_pair_of(&wr) {
             drop(wr);

tokio/src/io/util/async_seek_ext.rs

@@ -69,7 +69,7 @@ cfg_io_util! {
 
         /// Creates a future which will rewind to the beginning of the stream.
         ///
-        /// This is convenience method, equivalent to to `self.seek(SeekFrom::Start(0))`.
+        /// This is convenience method, equivalent to `self.seek(SeekFrom::Start(0))`.
         fn rewind(&mut self) -> Seek<'_, Self>
         where
             Self: Unpin,

tokio/src/io/util/vec_with_initialized.rs

@@ -1,19 +1,18 @@
 use crate::io::ReadBuf;
 use std::mem::MaybeUninit;
 
-mod private {
-    pub trait Sealed {}
+/// Something that looks like a `Vec<u8>`.
+///
+/// # Safety
+///
+/// The implementor must guarantee that the vector returned by the
+/// `as_mut` and `as_mut` methods do not change from one call to
+/// another.
+pub(crate) unsafe trait VecU8: AsRef<Vec<u8>> + AsMut<Vec<u8>> {}
 
-    impl Sealed for Vec<u8> {}
-    impl Sealed for &mut Vec<u8> {}
-}
+unsafe impl VecU8 for Vec<u8> {}
+unsafe impl VecU8 for &mut Vec<u8> {}
 
-/// A sealed trait that constrains the generic type parameter in `VecWithInitialized<V>`.  That struct's safety relies
-/// on certain invariants upheld by `Vec<u8>`.
-pub(crate) trait VecU8: AsMut<Vec<u8>> + private::Sealed {}
-
-impl VecU8 for Vec<u8> {}
-impl VecU8 for &mut Vec<u8> {}
 /// This struct wraps a `Vec<u8>` or `&mut Vec<u8>`, combining it with a
 /// `num_initialized`, which keeps track of the number of initialized bytes
 /// in the unused capacity.
@@ -64,8 +63,8 @@ where
     }
 
     #[cfg(feature = "io-util")]
-    pub(crate) fn is_empty(&mut self) -> bool {
-        self.vec.as_mut().is_empty()
+    pub(crate) fn is_empty(&self) -> bool {
+        self.vec.as_ref().is_empty()
     }
 
     pub(crate) fn get_read_buf<'a>(&'a mut self) -> ReadBuf<'a> {

tokio/src/lib.rs

@@ -16,6 +16,7 @@
 ))]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 #![cfg_attr(docsrs, allow(unused_attributes))]
+#![cfg_attr(loom, allow(dead_code, unreachable_pub))]
 
 //! A runtime for writing reliable network applications without compromising speed.
 //!
@@ -114,7 +115,7 @@
 //! The [`tokio::sync`] module contains synchronization primitives to use when
 //! needing to communicate or share data. These include:
 //!
-//! * channels ([`oneshot`], [`mpsc`], and [`watch`]), for sending values
+//! * channels ([`oneshot`], [`mpsc`], [`watch`], and [`broadcast`]), for sending values
 //!   between tasks,
 //! * a non-blocking [`Mutex`], for controlling access to a shared, mutable
 //!   value,
@@ -130,6 +131,7 @@
 //! [`oneshot`]: crate::sync::oneshot
 //! [`mpsc`]: crate::sync::mpsc
 //! [`watch`]: crate::sync::watch
+//! [`broadcast`]: crate::sync::broadcast
 //!
 //! The [`tokio::time`] module provides utilities for tracking time and
 //! scheduling work. This includes functions for setting [timeouts][timeout] for
@@ -151,7 +153,7 @@
 //! provide the functionality you need.
 //!
 //! Using the runtime requires the "rt" or "rt-multi-thread" feature flags, to
-//! enable the basic [single-threaded scheduler][rt] and the [thread-pool
+//! enable the current-thread [single-threaded scheduler][rt] and the [multi-thread
 //! scheduler][rt-multi-thread], respectively. See the [`runtime` module
 //! documentation][rt-features] for details. In addition, the "macros" feature
 //! flag enables the `#[tokio::main]` and `#[tokio::test]` attributes.
@@ -308,7 +310,7 @@
 //! need.
 //!
 //! - `full`: Enables all features listed below except `test-util` and `tracing`.
-//! - `rt`: Enables `tokio::spawn`, the basic (current thread) scheduler,
+//! - `rt`: Enables `tokio::spawn`, the current-thread scheduler,
 //!         and non-scheduler utilities.
 //! - `rt-multi-thread`: Enables the heavier, multi-threaded, work-stealing scheduler.
 //! - `io-util`: Enables the IO based `Ext` traits.
@@ -346,9 +348,12 @@
 //!
 //! Likewise, some parts of the API are only available with the same flag:
 //!
-//! - [`task::JoinSet`]
 //! - [`task::Builder`]
-//!  
+//! - Some methods on [`task::JoinSet`]
+//! - [`runtime::RuntimeMetrics`]
+//! - [`runtime::Builder::unhandled_panic`]
+//! - [`task::Id`]
+//!
 //! This flag enables **unstable** features. The public API of these features
 //! may break in 1.x releases. To enable these features, the `--cfg
 //! tokio_unstable` argument must be passed to `rustc` when compiling. This
@@ -378,6 +383,39 @@
 //!
 //! [unstable features]: https://internals.rust-lang.org/t/feature-request-unstable-opt-in-non-transitive-crate-features/16193#why-not-a-crate-feature-2
 //! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section
+//!
+//! ## WASM support
+//!
+//! Tokio has some limited support for the WASM platform. Without the
+//! `tokio_unstable` flag, the following features are supported:
+//!
+//!  * `sync`
+//!  * `macros`
+//!  * `io-util`
+//!  * `rt`
+//!  * `time`
+//!
+//! Enabling any other feature (including `full`) will cause a compilation
+//! failure.
+//!
+//! The `time` module will only work on WASM platforms that have support for
+//! timers (e.g. wasm32-wasi). The timing functions will panic if used on a WASM
+//! platform that does not support timers.
+//!
+//! Note also that if the runtime becomes indefinitely idle, it will panic
+//! immediately instead of blocking forever. On platforms that don't support
+//! time, this means that the runtime can never be idle in any way.
+//!
+//! ### Unstable WASM support
+//!
+//! Tokio also has unstable support for some additional WASM features. This
+//! requires the use of the `tokio_unstable` flag.
+//!
+//! Using this flag enables the use of `tokio::net` on the wasm32-wasi target.
+//! However, not all methods are available on the networking types as WASI
+//! currently does not support the creation of new sockets from within WASM.
+//! Because of this, sockets must currently be created via the `FromRawFd`
+//! trait.
 
 // Test that pointer width is compatible. This asserts that e.g. usize is at
 // least 32 bits, which a lot of components in Tokio currently assumes.
@@ -392,6 +430,37 @@ compile_error! {
     "Tokio requires the platform pointer width to be 32, 64, or 128 bits"
 }
 
+// Ensure that our build script has correctly set cfg flags for wasm.
+//
+// Each condition is written all(a, not(b)). This should be read as
+// "if a, then we must also have b".
+#[cfg(any(
+    all(target_arch = "wasm32", not(tokio_wasm)),
+    all(target_arch = "wasm64", not(tokio_wasm)),
+    all(target_family = "wasm", not(tokio_wasm)),
+    all(target_os = "wasi", not(tokio_wasm)),
+    all(target_os = "wasi", not(tokio_wasi)),
+    all(target_os = "wasi", tokio_wasm_not_wasi),
+    all(tokio_wasm, not(any(target_arch = "wasm32", target_arch = "wasm64"))),
+    all(tokio_wasm_not_wasi, not(tokio_wasm)),
+    all(tokio_wasi, not(tokio_wasm))
+))]
+compile_error!("Tokio's build script has incorrectly detected wasm.");
+
+#[cfg(all(
+    not(tokio_unstable),
+    tokio_wasm,
+    any(
+        feature = "fs",
+        feature = "io-std",
+        feature = "net",
+        feature = "process",
+        feature = "rt-multi-thread",
+        feature = "signal"
+    )
+))]
+compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm.");
+
 // Includes re-exports used by macros.
 //
 // This module is not intended to be part of the public API. In general, any
@@ -416,12 +485,21 @@ cfg_process! {
     pub mod process;
 }
 
-#[cfg(any(feature = "net", feature = "fs", feature = "io-std"))]
+#[cfg(any(
+    feature = "fs",
+    feature = "io-std",
+    feature = "net",
+    all(windows, feature = "process"),
+))]
 mod blocking;
 
 cfg_rt! {
     pub mod runtime;
 }
+cfg_not_rt! {
+    // The `runtime` module is used when the IO or time driver is needed.
+    pub(crate) mod runtime;
+}
 
 pub(crate) mod coop;
 

tokio/src/loom/mocked.rs

@@ -38,3 +38,8 @@ pub(crate) mod sys {
         2
     }
 }
+
+pub(crate) mod thread {
+    pub use loom::lazy_static::AccessError;
+    pub use loom::thread::*;
+}

tokio/src/loom/std/mod.rs

@@ -102,7 +102,7 @@ pub(crate) mod thread {
 
     #[allow(unused_imports)]
     pub(crate) use std::thread::{
-        current, panicking, park, park_timeout, sleep, spawn, Builder, JoinHandle, LocalKey,
-        Result, Thread, ThreadId,
+        current, panicking, park, park_timeout, sleep, spawn, AccessError, Builder, JoinHandle,
+        LocalKey, Result, Thread, ThreadId,
     };
 }

tokio/src/macros/addr_of.rs

@@ -0,0 +1,53 @@
+//! This module defines a macro that lets you go from a raw pointer to a struct
+//! to a raw pointer to a field of the struct.
+
+#[cfg(not(tokio_no_addr_of))]
+macro_rules! generate_addr_of_methods {
+    (
+    impl<$($gen:ident)*> $struct_name:ty {$(
+        $(#[$attrs:meta])*
+        $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> {
+            &self$(.$field_name:tt)+
+        }
+    )*}
+    ) => {
+        impl<$($gen)*> $struct_name {$(
+            $(#[$attrs])*
+            $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> {
+                let me = me.as_ptr();
+                let field = ::std::ptr::addr_of_mut!((*me) $(.$field_name)+ );
+                ::core::ptr::NonNull::new_unchecked(field)
+            }
+        )*}
+    };
+}
+
+// The `addr_of_mut!` macro is only available for MSRV at least 1.51.0. This
+// version of the macro uses a workaround for older versions of rustc.
+#[cfg(tokio_no_addr_of)]
+macro_rules! generate_addr_of_methods {
+    (
+    impl<$($gen:ident)*> $struct_name:ty {$(
+        $(#[$attrs:meta])*
+        $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> {
+            &self$(.$field_name:tt)+
+        }
+    )*}
+    ) => {
+        impl<$($gen)*> $struct_name {$(
+            $(#[$attrs])*
+            $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> {
+                let me = me.as_ptr();
+                let me_u8 = me as *mut u8;
+
+                let field_offset = {
+                    let me_ref = &*me;
+                    let field_ref_u8 = (&me_ref $(.$field_name)+ ) as *const $field_type as *const u8;
+                    field_ref_u8.offset_from(me_u8)
+                };
+
+                ::core::ptr::NonNull::new_unchecked(me_u8.offset(field_offset).cast())
+            }
+        )*}
+    };
+}

tokio/src/macros/cfg.rs

@@ -61,6 +61,7 @@ macro_rules! cfg_fs {
     ($($item:item)*) => {
         $(
             #[cfg(feature = "fs")]
+            #[cfg(not(tokio_wasi))]
             #[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
             $item
         )*
@@ -69,7 +70,11 @@ macro_rules! cfg_fs {
 
 macro_rules! cfg_io_blocking {
     ($($item:item)*) => {
-        $( #[cfg(any(feature = "io-std", feature = "fs"))] $item )*
+        $( #[cfg(any(
+                feature = "io-std",
+                feature = "fs",
+                all(windows, feature = "process"),
+        ))] $item )*
     }
 }
 
@@ -78,12 +83,12 @@ macro_rules! cfg_io_driver {
         $(
             #[cfg(any(
                 feature = "net",
-                feature = "process",
+                all(unix, feature = "process"),
                 all(unix, feature = "signal"),
             ))]
             #[cfg_attr(docsrs, doc(cfg(any(
                 feature = "net",
-                feature = "process",
+                all(unix, feature = "process"),
                 all(unix, feature = "signal"),
             ))))]
             $item
@@ -96,7 +101,7 @@ macro_rules! cfg_io_driver_impl {
         $(
             #[cfg(any(
                 feature = "net",
-                feature = "process",
+                all(unix, feature = "process"),
                 all(unix, feature = "signal"),
             ))]
             $item
@@ -109,7 +114,7 @@ macro_rules! cfg_not_io_driver {
         $(
             #[cfg(not(any(
                 feature = "net",
-                feature = "process",
+                all(unix, feature = "process"),
                 all(unix, feature = "signal"),
             )))]
             $item
@@ -195,9 +200,19 @@ macro_rules! cfg_not_metrics {
     }
 }
 
-macro_rules! cfg_not_rt_and_metrics {
+macro_rules! cfg_not_rt_and_metrics_and_net {
     ($($item:item)*) => {
-        $( #[cfg(not(all(feature = "rt", all(tokio_unstable, not(loom)))))] $item )*
+        $( #[cfg(not(all(feature = "net", feature = "rt", all(tokio_unstable, not(loom)))))]$item )*
+    }
+}
+
+macro_rules! cfg_net_or_process {
+    ($($item:item)*) => {
+        $(
+            #[cfg(any(feature = "net", feature = "process"))]
+            #[cfg_attr(docsrs, doc(cfg(any(feature = "net", feature = "process"))))]
+            $item
+        )*
     }
 }
 
@@ -237,6 +252,7 @@ macro_rules! cfg_process {
             #[cfg(feature = "process")]
             #[cfg_attr(docsrs, doc(cfg(feature = "process")))]
             #[cfg(not(loom))]
+            #[cfg(not(tokio_wasi))]
             $item
         )*
     }
@@ -265,6 +281,7 @@ macro_rules! cfg_signal {
             #[cfg(feature = "signal")]
             #[cfg_attr(docsrs, doc(cfg(feature = "signal")))]
             #[cfg(not(loom))]
+            #[cfg(not(tokio_wasi))]
             $item
         )*
     }
@@ -324,7 +341,7 @@ macro_rules! cfg_not_rt {
 macro_rules! cfg_rt_multi_thread {
     ($($item:item)*) => {
         $(
-            #[cfg(feature = "rt-multi-thread")]
+            #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
             #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
             $item
         )*
@@ -441,7 +458,8 @@ macro_rules! cfg_has_atomic_u64 {
                     target_arch = "arm",
                     target_arch = "mips",
                     target_arch = "powerpc",
-                    target_arch = "riscv32"
+                    target_arch = "riscv32",
+                    tokio_wasm
                     )))]
             $item
         )*
@@ -455,9 +473,28 @@ macro_rules! cfg_not_has_atomic_u64 {
                     target_arch = "arm",
                     target_arch = "mips",
                     target_arch = "powerpc",
-                    target_arch = "riscv32"
+                    target_arch = "riscv32",
+                    tokio_wasm
                     ))]
             $item
         )*
     }
 }
+
+macro_rules! cfg_not_wasi {
+    ($($item:item)*) => {
+        $(
+            #[cfg(not(tokio_wasi))]
+            $item
+        )*
+    }
+}
+
+macro_rules! cfg_is_wasm_not_wasi {
+    ($($item:item)*) => {
+        $(
+            #[cfg(tokio_wasm_not_wasi)]
+            $item
+        )*
+    }
+}

tokio/src/macros/join.rs

@@ -12,7 +12,7 @@
 /// for **all** branches complete regardless if any complete with `Err`. Use
 /// [`try_join!`] to return early when `Err` is encountered.
 ///
-/// [`try_join!`]: macro@try_join
+/// [`try_join!`]: crate::try_join
 ///
 /// # Notes
 ///
@@ -60,6 +60,9 @@ macro_rules! join {
         // normalization is complete.
         ( $($count:tt)* )
 
+        // The expression `0+1+1+ ... +1` equal to the number of branches.
+        ( $($total:tt)* )
+
         // Normalized join! branches
         $( ( $($skip:tt)* ) $e:expr, )*
 
@@ -71,22 +74,54 @@ macro_rules! join {
         // the requirement of `Pin::new_unchecked` called below.
         let mut futures = ( $( maybe_done($e), )* );
 
+        // Each time the future created by poll_fn is polled, a different future will be polled first
+        // to ensure every future passed to join! gets a chance to make progress even if
+        // one of the futures consumes the whole budget.
+        //
+        // This is number of futures that will be skipped in the first loop
+        // iteration the next time.
+        let mut skip_next_time: u32 = 0;
+
         poll_fn(move |cx| {
+            const COUNT: u32 = $($total)*;
+
             let mut is_pending = false;
 
+            let mut to_run = COUNT;
+
+            // The number of futures that will be skipped in the first loop iteration.
+            let mut skip = skip_next_time;
+
+            skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 };
+
+            // This loop runs twice and the first `skip` futures
+            // are not polled in the first iteration.
+            loop {
             $(
-                // Extract the future for this branch from the tuple.
-                let ( $($skip,)* fut, .. ) = &mut futures;
+                if skip == 0 {
+                    if to_run == 0 {
+                        // Every future has been polled
+                        break;
+                    }
+                    to_run -= 1;
 
-                // Safety: future is stored on the stack above
-                // and never moved.
-                let mut fut = unsafe { Pin::new_unchecked(fut) };
+                    // Extract the future for this branch from the tuple.
+                    let ( $($skip,)* fut, .. ) = &mut futures;
 
-                // Try polling
-                if fut.poll(cx).is_pending() {
-                    is_pending = true;
+                    // Safety: future is stored on the stack above
+                    // and never moved.
+                    let mut fut = unsafe { Pin::new_unchecked(fut) };
+
+                    // Try polling
+                    if fut.poll(cx).is_pending() {
+                        is_pending = true;
+                    }
+                } else {
+                    // Future skipped, one less future to skip in the next iteration
+                    skip -= 1;
                 }
             )*
+            }
 
             if is_pending {
                 Pending
@@ -107,13 +142,13 @@ macro_rules! join {
 
     // ===== Normalize =====
 
-    (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
-        $crate::join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*)
+    (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
+        $crate::join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*)
     };
 
     // ===== Entry point =====
 
     ( $($e:expr),* $(,)?) => {
-        $crate::join!(@{ () } $($e,)*)
+        $crate::join!(@{ () (0) } $($e,)*)
     };
 }

tokio/src/macros/mod.rs

@@ -15,6 +15,9 @@ mod ready;
 #[macro_use]
 mod thread_local;
 
+#[macro_use]
+mod addr_of;
+
 cfg_trace! {
     #[macro_use]
     mod trace;

tokio/src/macros/scoped_tls.rs

@@ -10,7 +10,7 @@ macro_rules! scoped_thread_local {
         $vis static $name: $crate::macros::scoped_tls::ScopedKey<$ty>
             = $crate::macros::scoped_tls::ScopedKey {
                 inner: {
-                    thread_local!(static FOO: ::std::cell::Cell<*const ()> = {
+                    thread_local!(static FOO: ::std::cell::Cell<*const ()> = const {
                         std::cell::Cell::new(::std::ptr::null())
                     });
                     &FOO

tokio/src/macros/thread_local.rs

@@ -1,4 +1,30 @@
 #[cfg(all(loom, test))]
 macro_rules! thread_local {
+    ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => {
+        loom::thread_local! {
+            $(#[$attrs])*
+            $vis static $name: $ty = $expr;
+        }
+    };
+
     ($($tts:tt)+) => { loom::thread_local!{ $($tts)+ } }
 }
+
+#[cfg(not(tokio_no_const_thread_local))]
+#[cfg(not(all(loom, test)))]
+macro_rules! thread_local {
+    ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } }
+}
+
+#[cfg(tokio_no_const_thread_local)]
+#[cfg(not(all(loom, test)))]
+macro_rules! thread_local {
+    ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => {
+        ::std::thread_local! {
+            $(#[$attrs])*
+            $vis static $name: $ty = $expr;
+        }
+    };
+
+    ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } }
+}

tokio/src/macros/try_join.rs

@@ -106,6 +106,9 @@ macro_rules! try_join {
         // normalization is complete.
         ( $($count:tt)* )
 
+        // The expression `0+1+1+ ... +1` equal to the number of branches.
+        ( $($total:tt)* )
+
         // Normalized try_join! branches
         $( ( $($skip:tt)* ) $e:expr, )*
 
@@ -117,24 +120,56 @@ macro_rules! try_join {
         // the requirement of `Pin::new_unchecked` called below.
         let mut futures = ( $( maybe_done($e), )* );
 
+        // Each time the future created by poll_fn is polled, a different future will be polled first
+        // to ensure every future passed to join! gets a chance to make progress even if
+        // one of the futures consumes the whole budget.
+        //
+        // This is number of futures that will be skipped in the first loop
+        // iteration the next time.
+        let mut skip_next_time: u32 = 0;
+
         poll_fn(move |cx| {
+            const COUNT: u32 = $($total)*;
+
             let mut is_pending = false;
 
+            let mut to_run = COUNT;
+
+            // The number of futures that will be skipped in the first loop iteration
+            let mut skip = skip_next_time;
+
+            skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 };
+
+            // This loop runs twice and the first `skip` futures
+            // are not polled in the first iteration.
+            loop {
             $(
-                // Extract the future for this branch from the tuple.
-                let ( $($skip,)* fut, .. ) = &mut futures;
+                if skip == 0 {
+                    if to_run == 0 {
+                        // Every future has been polled
+                        break;
+                    }
+                    to_run -= 1;
 
-                // Safety: future is stored on the stack above
-                // and never moved.
-                let mut fut = unsafe { Pin::new_unchecked(fut) };
+                    // Extract the future for this branch from the tuple.
+                    let ( $($skip,)* fut, .. ) = &mut futures;
 
-                // Try polling
-                if fut.as_mut().poll(cx).is_pending() {
-                    is_pending = true;
-                } else if fut.as_mut().output_mut().expect("expected completed future").is_err() {
-                    return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap()))
+                    // Safety: future is stored on the stack above
+                    // and never moved.
+                    let mut fut = unsafe { Pin::new_unchecked(fut) };
+
+                    // Try polling
+                    if fut.as_mut().poll(cx).is_pending() {
+                        is_pending = true;
+                    } else if fut.as_mut().output_mut().expect("expected completed future").is_err() {
+                        return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap()))
+                    }
+                } else {
+                    // Future skipped, one less future to skip in the next iteration
+                    skip -= 1;
                 }
             )*
+            }
 
             if is_pending {
                 Pending
@@ -159,13 +194,13 @@ macro_rules! try_join {
 
     // ===== Normalize =====
 
-    (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
-        $crate::try_join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*)
+    (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
+      $crate::try_join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*)
     };
 
     // ===== Entry point =====
 
     ( $($e:expr),* $(,)?) => {
-        $crate::try_join!(@{ () } $($e,)*)
+        $crate::try_join!(@{ () (0) } $($e,)*)
     };
 }