chore: prepare Tokio v1.19.1 (#4750)

Co-authored-by: Taiki Endo <[email protected]>

.cargo/audit.toml

@@ -2,6 +2,7 @@
 
 [advisories]
 ignore = [
-    # https://github.com/tokio-rs/tokio/issues/4177
-    "RUSTSEC-2020-0159",
+    # We depend on nix 0.22 only via mio-aio, a dev-dependency.
+    # https://github.com/tokio-rs/tokio/pull/4255#issuecomment-974786349
+    "RUSTSEC-2021-0119",
 ]

.cargo/config

@@ -0,0 +1,2 @@
+# [build]
+# rustflags = ["--cfg", "tokio_unstable"]

.circleci/config.yml

@@ -0,0 +1,25 @@
+version: 2.1        
+jobs:
+  test-arm:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+    environment:
+      # Change to pin rust versino
+      RUST_STABLE: stable
+    steps:
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup.sh
+            chmod +x rustup.sh
+            ./rustup.sh -y --default-toolchain $RUST_STABLE
+            source "$HOME"/.cargo/env
+      # Only run Tokio tests
+      - run: cargo test --all-features -p tokio
+
+workflows:
+  ci:
+    jobs:
+      - test-arm

.cirrus.yml

@@ -1,6 +1,8 @@
 freebsd_instance:
-  image: freebsd-12-2-release-amd64
+  image: freebsd-12-3-release-amd64
 env:
+  RUST_STABLE: stable
+  RUST_NIGHTLY: nightly-2022-03-21
   RUSTFLAGS: -D warnings
 
 # Test FreeBSD in a full VM on cirrus-ci.com.  Test the i686 target too, in the
@@ -12,7 +14,7 @@ task:
   setup_script:
     - pkg install -y bash curl
     - curl https://sh.rustup.rs -sSf --output rustup.sh
-    - sh rustup.sh -y --profile minimal --default-toolchain stable
+    - sh rustup.sh -y --profile minimal --default-toolchain $RUST_STABLE
     - . $HOME/.cargo/env
     - |
       echo "~~~~ rustc --version ~~~~"
@@ -24,12 +26,12 @@ task:
 task:
   name: FreeBSD docs
   env:
-    RUSTFLAGS: --cfg docsrs
-    RUSTDOCFLAGS: --cfg docsrs -Dwarnings
+    RUSTFLAGS: --cfg docsrs --cfg tokio_unstable
+    RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable -Dwarnings
   setup_script:
     - pkg install -y bash curl
     - curl https://sh.rustup.rs -sSf --output rustup.sh
-    - sh rustup.sh -y --profile minimal --default-toolchain nightly-2021-10-25
+    - sh rustup.sh -y --profile minimal --default-toolchain $RUST_NIGHTLY
     - . $HOME/.cargo/env
     - |
       echo "~~~~ rustc --version ~~~~"
@@ -43,7 +45,7 @@ task:
   setup_script:
     - pkg install -y bash curl
     - curl https://sh.rustup.rs -sSf --output rustup.sh
-    - sh rustup.sh -y --profile minimal --default-toolchain stable
+    - sh rustup.sh -y --profile minimal --default-toolchain $RUST_STABLE
     - . $HOME/.cargo/env
     - rustup target add i686-unknown-freebsd
     - |

.clippy.toml

@@ -1 +1 @@
-msrv = "1.45"
+msrv = "1.49"

.github/labeler.yml

@@ -1,9 +1,8 @@
 
 R-loom:
-- ./tokio/src/sync/*
-- ./tokio/src/sync/**/*
-- ./tokio-util/src/sync/*
-- ./tokio-util/src/sync/**/*
-- ./tokio/src/runtime/*
-- ./tokio/src/runtime/**/*
-
+- tokio/src/sync/*
+- tokio/src/sync/**/*
+- tokio-util/src/sync/*
+- tokio-util/src/sync/**/*
+- tokio/src/runtime/*
+- tokio/src/runtime/**/*

.github/workflows/audit.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Audit Check
         uses: actions-rs/audit-check@v1

.github/workflows/ci.yml

@@ -9,8 +9,23 @@ name: CI
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
-  nightly: nightly-2021-10-25
-  minrust: 1.45.2
+  # Change to specific Rust release to pin
+  rust_stable: stable
+  rust_nightly: nightly-2022-04-18
+  rust_clippy: 1.52.0
+  # When updating this, also update:
+  # - README.md
+  # - tokio/README.md
+  # - CONTRIBUTING.md
+  # - tokio/Cargo.toml
+  # - tokio-util/Cargo.toml
+  # - tokio-test/Cargo.toml
+  # - tokio-stream/Cargo.toml
+  rust_min: 1.49.0
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   # Depends on all action sthat are required for a "successful" CI run.
@@ -20,6 +35,7 @@ jobs:
     needs:
       - test
       - test-unstable
+      - test-parking_lot
       - miri
       - cross
       - features
@@ -30,6 +46,8 @@ jobs:
       - valgrind
       - loom-compile
       - check-readme
+      - test-hyper
+      - wasm32-unknown-unknown
     steps:
       - run: exit 0
 
@@ -43,7 +61,12 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
       - name: Install Rust
         run: rustup update stable
       - uses: Swatinem/rust-cache@v1
@@ -77,13 +100,40 @@ jobs:
         # bench.yml workflow runs benchmarks only on linux.
         if: startsWith(matrix.os, 'ubuntu')
 
+  test-parking_lot:
+    # The parking_lot crate has a feature called send_guard which changes when
+    # some of its types are Send. Tokio has some measures in place to prevent
+    # this from affecting when Tokio types are Send, and this test exists to
+    # ensure that those measures are working.
+    #
+    # This relies on the potentially affected Tokio type being listed in
+    # `tokio/tokio/tests/async_send_sync.rs`.
+    name: compile tests with parking lot send guards
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
+      - uses: Swatinem/rust-cache@v1
+      - name: Enable parking_lot send_guard feature
+        # Inserts the line "plsend = ["parking_lot/send_guard"]" right after [features]
+        run: sed -i '/\[features\]/a plsend = ["parking_lot/send_guard"]' tokio/Cargo.toml
+      - name: Compile tests with all features enabled
+        run: cargo build --workspace --all-features --tests
+
   valgrind:
     name: valgrind
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
       - uses: Swatinem/rust-cache@v1
 
       - name: Install Valgrind
@@ -119,54 +169,64 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
       - uses: Swatinem/rust-cache@v1
-
       # Run `tokio` with "unstable" cfg flag.
       - name: test tokio full --cfg unstable
         run: cargo test --all-features
         working-directory: tokio
         env:
           RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+          # in order to run doctests for unstable features, we must also pass
+          # the unstable cfg to RustDoc
+          RUSTDOCFLAGS: --cfg tokio_unstable
 
   miri:
     name: miri
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.nightly }}
+          toolchain: ${{ env.rust_nightly }}
+          components: miri
           override: true
       - uses: Swatinem/rust-cache@v1
-      - name: Install Miri
-        run: |
-          set -e
-          rustup component add miri
-          cargo miri setup
-          rm -rf tokio/tests
-
       - name: miri
-        run: cargo miri test --features rt,rt-multi-thread,sync task
+        # Many of tests in tokio/tests and doctests use #[tokio::test] or
+        # #[tokio::main] that calls epoll_create1 that Miri does not support.
+        run: cargo miri test --features full --lib --no-fail-fast
         working-directory: tokio
-  san:
-    name: san
+        env:
+          MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-tag-raw-pointers
+          PROPTEST_CASES: 10
+
+  asan:
+    name: asan
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install llvm
+        # Required to resolve symbols in sanitizer output
+        run: sudo apt-get install -y llvm
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.nightly }}
+          toolchain: ${{ env.rust_nightly }}
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: asan
-        run: cargo test --all-features --target x86_64-unknown-linux-gnu --lib -- --test-threads 1
-        working-directory: tokio
+        run: cargo test --workspace --all-features --target x86_64-unknown-linux-gnu --tests -- --test-threads 1
         env:
           RUSTFLAGS: -Z sanitizer=address
-          ASAN_OPTIONS: detect_leaks=0
+          # Ignore `trybuild` errors as they are irrelevant and flaky on nightly
+          TRYBUILD: overwrite
 
   cross:
     name: cross
@@ -179,11 +239,13 @@ jobs:
           - powerpc64-unknown-linux-gnu
           - mips-unknown-linux-gnu
           - arm-linux-androideabi
+          - mipsel-unknown-linux-musl
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: stable
+          toolchain: ${{ env.rust_stable }}
           target: ${{ matrix.target }}
           override: true
       - uses: Swatinem/rust-cache@v1
@@ -191,24 +253,31 @@ jobs:
         with:
           use-cross: true
           command: check
-          args: --workspace --target ${{ matrix.target }}
+          args: --workspace --all-features --target ${{ matrix.target }}
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: check
+          args: --workspace --all-features --target ${{ matrix.target }}
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
 
   features:
     name: features
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.nightly }}
+          toolchain: ${{ env.rust_nightly }}
+          target: ${{ matrix.target }}
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: Install cargo-hack
         run: cargo install cargo-hack
-
       - name: check --each-feature
         run: cargo hack check --all --each-feature -Z avoid-dev-deps
-
       # Try with unstable feature flags
       - name: check --each-feature --unstable
         run: cargo hack check --all --each-feature -Z avoid-dev-deps
@@ -219,10 +288,11 @@ jobs:
     name: minrust
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_min }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.minrust }}
+          toolchain: ${{ env.rust_min }}
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: "test --workspace --all-features"
@@ -232,10 +302,11 @@ jobs:
     name: minimal-versions
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.nightly }}
+          toolchain: ${{ env.rust_nightly }}
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: Install cargo-hack
@@ -248,18 +319,29 @@ jobs:
           # Update Cargo.lock to minimal version dependencies.
           cargo update -Z minimal-versions
           cargo hack check --all-features --ignore-private
+      - name: "check --all-features --unstable -Z minimal-versions"
+        env:
+          RUSTFLAGS: --cfg tokio_unstable -Dwarnings
+        run: |
+          # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update`
+          # from determining minimal versions based on dev-dependencies.
+          cargo hack --remove-dev-deps --workspace
+          # Update Cargo.lock to minimal version dependencies.
+          cargo update -Z minimal-versions
+          cargo hack check --all-features --ignore-private
 
   fmt:
     name: fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          override: true
+          components: rustfmt
       - uses: Swatinem/rust-cache@v1
-      - name: Install rustfmt
-        run: rustup component add rustfmt
-
       # Check fmt
       - name: "rustfmt --check"
         # Workaround for rust-lang/cargo#7732
@@ -273,13 +355,14 @@ jobs:
     name: clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update 1.52.1 && rustup default 1.52.1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_clippy }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_clippy }}
+          override: true
+          components: clippy
       - uses: Swatinem/rust-cache@v1
-      - name: Install clippy
-        run: rustup component add clippy
-
       # Run clippy
       - name: "clippy --all"
         run: cargo clippy --all --tests --all-features
@@ -288,25 +371,29 @@ jobs:
     name: docs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_nightly }}
+        uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ env.nightly }}
+          toolchain: ${{ env.rust_nightly }}
           override: true
       - uses: Swatinem/rust-cache@v1
       - name: "doc --lib --all-features"
         run: cargo doc --lib --no-deps --all-features --document-private-items
         env:
-          RUSTFLAGS: --cfg docsrs
-          RUSTDOCFLAGS: --cfg docsrs -Dwarnings
+          RUSTFLAGS: --cfg docsrs --cfg tokio_unstable
+          RUSTDOCFLAGS: --cfg docsrs --cfg tokio_unstable -Dwarnings
 
   loom-compile:
     name: build loom tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          override: true
       - uses: Swatinem/rust-cache@v1
       - name: build --cfg loom
         run: cargo test --no-run --lib --features full
@@ -318,10 +405,61 @@ jobs:
     name: Check README
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Verify that both READMEs are identical
         run: diff README.md tokio/README.md
 
       - name: Verify that Tokio version is up to date in README
         working-directory: tokio
         run: grep -q "$(sed '/^version = /!d' Cargo.toml | head -n1)" README.md
+
+  test-hyper:
+    name: Test hyper
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - name: Test hyper
+        run: |
+          set -x
+          git clone https://github.com/hyperium/hyper.git
+          cd hyper
+          # checkout the latest release because HEAD maybe contains breakage.
+          tag=$(git describe --abbrev=0 --tags)
+          git checkout "${tag}"
+          echo '[workspace]' >>Cargo.toml
+          echo '[patch.crates-io]' >>Cargo.toml
+          echo 'tokio = { path = "../tokio" }' >>Cargo.toml
+          echo 'tokio-util = { path = "../tokio-util" }' >>Cargo.toml
+          echo 'tokio-stream = { path = "../tokio-stream" }' >>Cargo.toml
+          echo 'tokio-test = { path = "../tokio-test" }' >>Cargo.toml
+          git diff
+          cargo test --features full
+
+  wasm32-unknown-unknown:
+    name: test tokio for wasm32-unknown-unknown
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_stable }}
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - name: test tokio
+        run: wasm-pack test --node -- --features "macros sync"
+        working-directory: tokio

.github/workflows/labeler.yml

@@ -11,3 +11,4 @@ jobs:
     - uses: actions/labeler@v3
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        sync-labels: true

.github/workflows/loom.yml

@@ -10,6 +10,8 @@ name: Loom
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
+  # Change to specific Rust release to pin
+  rust_stable: stable
 
 jobs:
   loom:
@@ -27,9 +29,12 @@ jobs:
           - loom_pool::group_d
           - time::driver
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
       - uses: Swatinem/rust-cache@v1
       - name: loom ${{ matrix.scope }}
         run: cargo test --lib --release --features full -- --nocapture $SCOPE

.github/workflows/pr-audit.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install cargo-audit
         uses: actions-rs/cargo@v1

.github/workflows/stress-test.yml

@@ -5,6 +5,12 @@ on:
     branches:
       - master
 
+env:
+  RUSTFLAGS: -Dwarnings
+  RUST_BACKTRACE: 1
+  # Change to specific Rust release to pin
+  rust_stable: stable
+
 jobs:
   stess-test:
     name: Stress Test
@@ -14,9 +20,12 @@ jobs:
         stress-test:
           - simple_echo_tcp
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Rust
-        run: rustup update stable
+      - uses: actions/checkout@v3
+      - name: Install Rust ${{ env.rust_stable }}
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ env.rust_stable }}
+            override: true
       - uses: Swatinem/rust-cache@v1
       - name: Install Valgrind
         run: |

.gitignore

@@ -1,2 +1,4 @@
 target
 Cargo.lock
+
+.cargo/config.toml

CONTRIBUTING.md

@@ -131,6 +131,24 @@ cargo check --all-features
 cargo test --all-features
 ```
 
+Clippy must be run using the MSRV, so Tokio can avoid having to `#[allow]` new
+lints whose fixes would be incompatible with the current MSRV:
+
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
+```
+cargo +1.49.0 clippy --all-features
+```
+
 When building documentation normally, the markers that list the features
 required for various parts of Tokio are missing. To build the documentation
 correctly, use this command:
@@ -139,6 +157,14 @@ correctly, use this command:
 RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features
 ```
 
+To build documentation including Tokio's unstable features, it is necessary to
+pass `--cfg tokio_unstable` to both RustDoc *and* rustc. To build the
+documentation for unstable features, use this command:
+
+```
+RUSTDOCFLAGS="--cfg docsrs --cfg tokio_unstable" RUSTFLAGS="--cfg tokio_unstable" cargo +nightly doc --all-features
+```
+
 There is currently a [bug in cargo] that means documentation cannot be built
 from the root of the workspace. If you `cd` into the `tokio` subdirectory the
 command shown above will work.
@@ -165,6 +191,12 @@ LOOM_MAX_PREEMPTIONS=1 RUSTFLAGS="--cfg loom" \
     cargo test --lib --release --features full -- --test-threads=1 --nocapture
 ```
 
+You can run miri tests with
+```
+MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-tag-raw-pointers" PROPTEST_CASES=10 \
+    cargo +nightly miri test --features full --lib
+```
+
 ### Tests
 
 If the change being proposed alters code (as opposed to only documentation for

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.13.0", features = ["full"] }
+tokio = { version = "1.19.1", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -163,9 +163,21 @@ several other libraries, including:
 
 ## Supported Rust Versions
 
-Tokio is built against the latest stable release. The minimum supported version
-is 1.45.  The current Tokio version is not guaranteed to build on Rust versions
-earlier than the minimum supported version.
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- CONTRIBUTING.md
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
+Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
+least** 6 months. When increasing the MSRV, the new Rust version must have been
+released at least six months ago. The current MSRV is 1.49.0.
 
 ## Release schedule
 
@@ -180,17 +192,18 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.8.x` - LTS release until February 2022.
+ * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until January 2023
 
 Each LTS release will continue to receive backported fixes for at least half a
 year. If you wish to use a fixed minor release in your project, we recommend
 that you use an LTS release.
 
 To use a fixed minor version, you can specify the version with a tilde. For
-example, to specify that you wish to use the newest `1.8.x` patch release, you
+example, to specify that you wish to use the newest `1.14.x` patch release, you
 can use the following dependency specification:
 ```text
-tokio = { version = "~1.8", features = [...] }
+tokio = { version = "~1.14", features = [...] }
 ```
 
 ## License

benches/Cargo.toml

@@ -9,7 +9,7 @@ tokio = { version = "1.5.0", path = "../tokio", features = ["full"] }
 bencher = "0.1.5"
 
 [dev-dependencies]
-tokio-util = { version = "0.6.6", path = "../tokio-util", features = ["full"] }
+tokio-util = { version = "0.7.0", path = "../tokio-util", features = ["full"] }
 tokio-stream = { path = "../tokio-stream" }
 
 [target.'cfg(unix)'.dependencies]

benches/fs.rs

@@ -21,7 +21,7 @@ fn rt() -> tokio::runtime::Runtime {
 const BLOCK_COUNT: usize = 1_000;
 
 const BUFFER_SIZE: usize = 4096;
-const DEV_ZERO: &'static str = "/dev/zero";
+const DEV_ZERO: &str = "/dev/zero";
 
 fn async_read_codec(b: &mut Bencher) {
     let rt = rt();

examples/Cargo.toml

@@ -5,14 +5,14 @@ publish = false
 edition = "2018"
 
 # If you copy one of the examples into a new project, you should be using
-# [dependencies] instead.
+# [dependencies] instead, and delete the **path**.
 [dev-dependencies]
-tokio = { version = "1.0.0", path = "../tokio",features = ["full", "tracing"] }
-tokio-util = { version = "0.6.3", path = "../tokio-util",features = ["full"] }
+tokio = { version = "1.0.0", path = "../tokio", features = ["full", "tracing"] }
+tokio-util = { version = "0.7.0", path = "../tokio-util", features = ["full"] }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 
 tracing = "0.1"
-tracing-subscriber = { version = "0.2.7", default-features = false, features = ["fmt", "ansi", "env-filter", "chrono", "tracing-log"] }
+tracing-subscriber = { version = "0.3.1", default-features = false, features = ["fmt", "ansi", "env-filter", "tracing-log"] }
 bytes = "1.0.0"
 futures = { version = "0.3.0", features = ["thread-pool"]}
 http = "0.2"

examples/tinydb.rs

@@ -149,7 +149,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 }
 
 fn handle_request(line: &str, db: &Arc<Database>) -> Response {
-    let request = match Request::parse(&line) {
+    let request = match Request::parse(line) {
         Ok(req) => req,
         Err(e) => return Response::Error { msg: e },
     };

netlify.toml

@@ -0,0 +1,16 @@
+[build]
+  command = """
+    rustup install nightly --profile minimal && cargo doc --no-deps --all-features
+    """
+  publish = "target/doc"
+
+[build.environment]
+  RUSTDOCFLAGS="""
+    --cfg docsrs \
+    --cfg tokio_unstable \
+    """
+  RUSTFLAGS="--cfg tokio_unstable --cfg docsrs"
+
+[[redirects]]
+  from = "/"
+  to = "/tokio"

tests-build/tests/fail/macros_core_no_default.stderr

@@ -1,5 +1,5 @@
 error: The default runtime flavor is `multi_thread`, but the `rt-multi-thread` feature is disabled.
- --> $DIR/macros_core_no_default.rs:3:1
+ --> tests/fail/macros_core_no_default.rs:3:1
   |
 3 | #[tokio::main]
   | ^^^^^^^^^^^^^^

tests-build/tests/fail/macros_invalid_input.rs

@@ -1,3 +1,5 @@
+#![deny(duplicate_macro_attributes)]
+
 use tests_build::tokio;
 
 #[tokio::main]
@@ -33,6 +35,15 @@ async fn test_worker_threads_not_int() {}
 #[tokio::test(flavor = "current_thread", worker_threads = 4)]
 async fn test_worker_threads_and_current_thread() {}
 
+#[tokio::test(crate = 456)]
+async fn test_crate_not_ident_int() {}
+
+#[tokio::test(crate = "456")]
+async fn test_crate_not_ident_invalid() {}
+
+#[tokio::test(crate = "abc::edf")]
+async fn test_crate_not_ident_path() {}
+
 #[tokio::test]
 #[test]
 async fn test_has_second_test_attr() {}

tests-build/tests/fail/macros_invalid_input.stderr

@@ -1,71 +1,101 @@
 error: the `async` keyword is missing from the function declaration
- --> $DIR/macros_invalid_input.rs:4:1
+ --> $DIR/macros_invalid_input.rs:6:1
   |
-4 | fn main_is_not_async() {}
+6 | fn main_is_not_async() {}
   | ^^
 
-error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
- --> $DIR/macros_invalid_input.rs:6:15
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
+ --> $DIR/macros_invalid_input.rs:8:15
   |
-6 | #[tokio::main(foo)]
+8 | #[tokio::main(foo)]
   |               ^^^
 
 error: Must have specified ident
- --> $DIR/macros_invalid_input.rs:9:15
-  |
-9 | #[tokio::main(threadpool::bar)]
-  |               ^^^^^^^^^^^^^^^
+  --> $DIR/macros_invalid_input.rs:11:15
+   |
+11 | #[tokio::main(threadpool::bar)]
+   |               ^^^^^^^^^^^^^^^
 
 error: the `async` keyword is missing from the function declaration
-  --> $DIR/macros_invalid_input.rs:13:1
+  --> $DIR/macros_invalid_input.rs:15:1
    |
-13 | fn test_is_not_async() {}
+15 | fn test_is_not_async() {}
    | ^^
 
-error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
-  --> $DIR/macros_invalid_input.rs:15:15
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
+  --> $DIR/macros_invalid_input.rs:17:15
    |
-15 | #[tokio::test(foo)]
+17 | #[tokio::test(foo)]
    |               ^^^
 
-error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
-  --> $DIR/macros_invalid_input.rs:18:15
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`
+  --> $DIR/macros_invalid_input.rs:20:15
    |
-18 | #[tokio::test(foo = 123)]
+20 | #[tokio::test(foo = 123)]
    |               ^^^^^^^^^
 
 error: Failed to parse value of `flavor` as string.
-  --> $DIR/macros_invalid_input.rs:21:24
+  --> $DIR/macros_invalid_input.rs:23:24
    |
-21 | #[tokio::test(flavor = 123)]
+23 | #[tokio::test(flavor = 123)]
    |                        ^^^
 
 error: No such runtime flavor `foo`. The runtime flavors are `current_thread` and `multi_thread`.
-  --> $DIR/macros_invalid_input.rs:24:24
+  --> $DIR/macros_invalid_input.rs:26:24
    |
-24 | #[tokio::test(flavor = "foo")]
+26 | #[tokio::test(flavor = "foo")]
    |                        ^^^^^
 
 error: The `start_paused` option requires the `current_thread` runtime flavor. Use `#[tokio::test(flavor = "current_thread")]`
-  --> $DIR/macros_invalid_input.rs:27:55
+  --> $DIR/macros_invalid_input.rs:29:55
    |
-27 | #[tokio::test(flavor = "multi_thread", start_paused = false)]
+29 | #[tokio::test(flavor = "multi_thread", start_paused = false)]
    |                                                       ^^^^^
 
 error: Failed to parse value of `worker_threads` as integer.
-  --> $DIR/macros_invalid_input.rs:30:57
+  --> $DIR/macros_invalid_input.rs:32:57
    |
-30 | #[tokio::test(flavor = "multi_thread", worker_threads = "foo")]
+32 | #[tokio::test(flavor = "multi_thread", worker_threads = "foo")]
    |                                                         ^^^^^
 
 error: The `worker_threads` option requires the `multi_thread` runtime flavor. Use `#[tokio::test(flavor = "multi_thread")]`
-  --> $DIR/macros_invalid_input.rs:33:59
+  --> $DIR/macros_invalid_input.rs:35:59
    |
-33 | #[tokio::test(flavor = "current_thread", worker_threads = 4)]
+35 | #[tokio::test(flavor = "current_thread", worker_threads = 4)]
    |                                                           ^
 
-error: second test attribute is supplied
-  --> $DIR/macros_invalid_input.rs:37:1
+error: Failed to parse value of `crate` as ident.
+  --> $DIR/macros_invalid_input.rs:38:23
    |
-37 | #[test]
+38 | #[tokio::test(crate = 456)]
+   |                       ^^^
+
+error: Failed to parse value of `crate` as ident: "456"
+  --> $DIR/macros_invalid_input.rs:41:23
+   |
+41 | #[tokio::test(crate = "456")]
+   |                       ^^^^^
+
+error: Failed to parse value of `crate` as ident: "abc::edf"
+  --> $DIR/macros_invalid_input.rs:44:23
+   |
+44 | #[tokio::test(crate = "abc::edf")]
+   |                       ^^^^^^^^^^
+
+error: second test attribute is supplied
+  --> $DIR/macros_invalid_input.rs:48:1
+   |
+48 | #[test]
    | ^^^^^^^
+
+error: duplicated attribute
+  --> $DIR/macros_invalid_input.rs:48:1
+   |
+48 | #[test]
+   | ^^^^^^^
+   |
+note: the lint level is defined here
+  --> $DIR/macros_invalid_input.rs:1:9
+   |
+1  | #![deny(duplicate_macro_attributes)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^

tests-build/tests/fail/macros_type_mismatch.rs

@@ -23,4 +23,13 @@ async fn extra_semicolon() -> Result<(), ()> {
     Ok(());
 }
 
+// https://github.com/tokio-rs/tokio/issues/4635
+#[allow(redundant_semicolons)]
+#[rustfmt::skip]
+#[tokio::main]
+async fn issue_4635() {
+    return 1;
+    ;
+}
+
 fn main() {}

tests-build/tests/fail/macros_type_mismatch.stderr

@@ -1,25 +1,19 @@
 error[E0308]: mismatched types
  --> $DIR/macros_type_mismatch.rs:5:5
   |
+4 | async fn missing_semicolon_or_return_type() {
+  |                                             - help: a return type might be missing here: `-> _`
 5 |     Ok(())
   |     ^^^^^^ expected `()`, found enum `Result`
   |
   = note: expected unit type `()`
                   found enum `Result<(), _>`
-help: consider using a semicolon here
-  |
-5 |     Ok(());
-  |           +
-help: try adding a return type
-  |
-4 | async fn missing_semicolon_or_return_type() -> Result<(), _> {
-  |                                             ++++++++++++++++
 
 error[E0308]: mismatched types
   --> $DIR/macros_type_mismatch.rs:10:5
    |
 9  | async fn missing_return_type() {
-   |                                - help: try adding a return type: `-> Result<(), _>`
+   |                                - help: a return type might be missing here: `-> _`
 10 |     return Ok(());
    |     ^^^^^^^^^^^^^^ expected `()`, found enum `Result`
    |
@@ -37,9 +31,17 @@ error[E0308]: mismatched types
    |
    = note:   expected enum `Result<(), ()>`
            found unit type `()`
-help: try using a variant of the expected enum
+help: try adding an expression at the end of the block
    |
-23 |     Ok(Ok(());)
+23 ~     Ok(());;
+24 +     Ok(())
    |
-23 |     Err(Ok(());)
+
+error[E0308]: mismatched types
+  --> $DIR/macros_type_mismatch.rs:32:5
    |
+30 | async fn issue_4635() {
+   |                       - help: try adding a return type: `-> i32`
+31 |     return 1;
+32 |     ;
+   |     ^ expected `()`, found integer

tests-integration/Cargo.toml

@@ -20,7 +20,7 @@ required-features = ["rt-process-signal"]
 # For mem check
 rt-net = ["tokio/rt", "tokio/rt-multi-thread", "tokio/net"]
 # For test-process-signal
-rt-process-signal = ["rt", "tokio/process", "tokio/signal"]
+rt-process-signal = ["rt-net", "tokio/process", "tokio/signal"]
 
 full = [
     "macros",

tokio-macros/CHANGELOG.md

@@ -1,3 +1,23 @@
+# 1.8.0 (June 4th, 2022)
+
+- macros: always emit return statement ([#4636])
+- macros: support setting a custom crate name for `#[tokio::main]` and `#[tokio::test]` ([#4613])
+
+[#4613]: https://github.com/tokio-rs/tokio/pull/4613
+[#4636]: https://github.com/tokio-rs/tokio/pull/4636
+
+# 1.7.0 (December 15th, 2021)
+
+- macros: address remaining `clippy::semicolon_if_nothing_returned` warning ([#4252])
+
+[#4252]: https://github.com/tokio-rs/tokio/pull/4252
+
+# 1.6.0 (November 16th, 2021)
+
+- macros: fix mut patterns in `select!` macro ([#4211])
+
+[#4211]: https://github.com/tokio-rs/tokio/pull/4211
+
 # 1.5.1 (October 29th, 2021)
 
 - macros: fix type resolution error in `#[tokio::main]` ([#4176])

tokio-macros/Cargo.toml

@@ -2,17 +2,15 @@
 name = "tokio-macros"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update doc url
-#   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-macros-1.0.x" git tag.
-version = "1.5.1"
+version = "1.8.0"
 edition = "2018"
+rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-macros/1.5.1/tokio_macros"
 description = """
 Tokio's proc macros.
 """

tokio-macros/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-macros/src/entry.rs

@@ -1,5 +1,5 @@
 use proc_macro::TokenStream;
-use proc_macro2::Span;
+use proc_macro2::{Ident, Span};
 use quote::{quote, quote_spanned, ToTokens};
 use syn::parse::Parser;
 
@@ -29,6 +29,7 @@ struct FinalConfig {
     flavor: RuntimeFlavor,
     worker_threads: Option<usize>,
     start_paused: Option<bool>,
+    crate_name: Option<String>,
 }
 
 /// Config used in case of the attribute not being able to build a valid config
@@ -36,6 +37,7 @@ const DEFAULT_ERROR_CONFIG: FinalConfig = FinalConfig {
     flavor: RuntimeFlavor::CurrentThread,
     worker_threads: None,
     start_paused: None,
+    crate_name: None,
 };
 
 struct Configuration {
@@ -45,6 +47,7 @@ struct Configuration {
     worker_threads: Option<(usize, Span)>,
     start_paused: Option<(bool, Span)>,
     is_test: bool,
+    crate_name: Option<String>,
 }
 
 impl Configuration {
@@ -59,6 +62,7 @@ impl Configuration {
             worker_threads: None,
             start_paused: None,
             is_test,
+            crate_name: None,
         }
     }
 
@@ -104,6 +108,15 @@ impl Configuration {
         Ok(())
     }
 
+    fn set_crate_name(&mut self, name: syn::Lit, span: Span) -> Result<(), syn::Error> {
+        if self.crate_name.is_some() {
+            return Err(syn::Error::new(span, "`crate` set multiple times."));
+        }
+        let name_ident = parse_ident(name, span, "crate")?;
+        self.crate_name = Some(name_ident.to_string());
+        Ok(())
+    }
+
     fn macro_name(&self) -> &'static str {
         if self.is_test {
             "tokio::test"
@@ -151,6 +164,7 @@ impl Configuration {
         };
 
         Ok(FinalConfig {
+            crate_name: self.crate_name.clone(),
             flavor,
             worker_threads,
             start_paused,
@@ -185,6 +199,27 @@ fn parse_string(int: syn::Lit, span: Span, field: &str) -> Result<String, syn::E
     }
 }
 
+fn parse_ident(lit: syn::Lit, span: Span, field: &str) -> Result<Ident, syn::Error> {
+    match lit {
+        syn::Lit::Str(s) => {
+            let err = syn::Error::new(
+                span,
+                format!(
+                    "Failed to parse value of `{}` as ident: \"{}\"",
+                    field,
+                    s.value()
+                ),
+            );
+            let path = s.parse::<syn::Path>().map_err(|_| err.clone())?;
+            path.get_ident().cloned().ok_or(err)
+        }
+        _ => Err(syn::Error::new(
+            span,
+            format!("Failed to parse value of `{}` as ident.", field),
+        )),
+    }
+}
+
 fn parse_bool(bool: syn::Lit, span: Span, field: &str) -> Result<bool, syn::Error> {
     match bool {
         syn::Lit::Bool(b) => Ok(b.value),
@@ -243,9 +278,15 @@ fn build_config(
                         let msg = "Attribute `core_threads` is renamed to `worker_threads`";
                         return Err(syn::Error::new_spanned(namevalue, msg));
                     }
+                    "crate" => {
+                        config.set_crate_name(
+                            namevalue.lit.clone(),
+                            syn::spanned::Spanned::span(&namevalue.lit),
+                        )?;
+                    }
                     name => {
                         let msg = format!(
-                            "Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`",
+                            "Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`",
                             name,
                         );
                         return Err(syn::Error::new_spanned(namevalue, msg));
@@ -275,7 +316,7 @@ fn build_config(
                         format!("The `{}` attribute requires an argument.", name)
                     }
                     name => {
-                        format!("Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`", name)
+                        format!("Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`, `crate`", name)
                     }
                 };
                 return Err(syn::Error::new_spanned(path, msg));
@@ -313,12 +354,16 @@ fn parse_knobs(mut input: syn::ItemFn, is_test: bool, config: FinalConfig) -> To
         (start, end)
     };
 
+    let crate_name = config.crate_name.as_deref().unwrap_or("tokio");
+
+    let crate_ident = Ident::new(crate_name, last_stmt_start_span);
+
     let mut rt = match config.flavor {
         RuntimeFlavor::CurrentThread => quote_spanned! {last_stmt_start_span=>
-            tokio::runtime::Builder::new_current_thread()
+            #crate_ident::runtime::Builder::new_current_thread()
         },
         RuntimeFlavor::Threaded => quote_spanned! {last_stmt_start_span=>
-            tokio::runtime::Builder::new_multi_thread()
+            #crate_ident::runtime::Builder::new_multi_thread()
         },
     };
     if let Some(v) = config.worker_threads {
@@ -338,29 +383,17 @@ fn parse_knobs(mut input: syn::ItemFn, is_test: bool, config: FinalConfig) -> To
 
     let body = &input.block;
     let brace_token = input.block.brace_token;
-    let (tail_return, tail_semicolon) = match body.stmts.last() {
-        Some(syn::Stmt::Semi(expr, _)) => match expr {
-            syn::Expr::Return(_) => (quote! { return }, quote! { ; }),
-            _ => match &input.sig.output {
-                syn::ReturnType::Type(_, ty) if matches!(&**ty, syn::Type::Tuple(ty) if ty.elems.is_empty()) =>
-                {
-                    (quote! {}, quote! { ; }) // unit
-                }
-                syn::ReturnType::Default => (quote! {}, quote! { ; }), // unit
-                syn::ReturnType::Type(..) => (quote! {}, quote! {}),   // ! or another
-            },
-        },
-        _ => (quote! {}, quote! {}),
-    };
     input.block = syn::parse2(quote_spanned! {last_stmt_end_span=>
         {
             let body = async #body;
-            #[allow(clippy::expect_used)]
-            #tail_return #rt
-                .enable_all()
-                .build()
-                .expect("Failed building the Runtime")
-                .block_on(body)#tail_semicolon
+            #[allow(clippy::expect_used, clippy::diverging_sub_expression)]
+            {
+                return #rt
+                    .enable_all()
+                    .build()
+                    .expect("Failed building the Runtime")
+                    .block_on(body);
+            }
         }
     })
     .expect("Parsing failure");

tokio-macros/src/lib.rs

@@ -5,7 +5,6 @@
     rust_2018_idioms,
     unreachable_pub
 )]
-#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
@@ -169,12 +168,32 @@ use proc_macro::TokenStream;
 ///
 /// Note that `start_paused` requires the `test-util` feature to be enabled.
 ///
-/// ### NOTE:
+/// ### Rename package
 ///
-/// If you rename the Tokio crate in your dependencies this macro will not work.
-/// If you must rename the current version of Tokio because you're also using an
-/// older version of Tokio, you _must_ make the current version of Tokio
-/// available as `tokio` in the module where this macro is expanded.
+/// ```rust
+/// use tokio as tokio1;
+///
+/// #[tokio1::main(crate = "tokio1")]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// use tokio as tokio1;
+///
+/// fn main() {
+///     tokio1::runtime::Builder::new_multi_thread()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
 #[proc_macro_attribute]
 #[cfg(not(test))] // Work around for rust-lang/rust#62127
 pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
@@ -214,12 +233,32 @@ pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
 /// }
 /// ```
 ///
-/// ### NOTE:
+/// ### Rename package
 ///
-/// If you rename the Tokio crate in your dependencies this macro will not work.
-/// If you must rename the current version of Tokio because you're also using an
-/// older version of Tokio, you _must_ make the current version of Tokio
-/// available as `tokio` in the module where this macro is expanded.
+/// ```rust
+/// use tokio as tokio1;
+///
+/// #[tokio1::main(crate = "tokio1")]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// use tokio as tokio1;
+///
+/// fn main() {
+///     tokio1::runtime::Builder::new_multi_thread()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
 #[proc_macro_attribute]
 #[cfg(not(test))] // Work around for rust-lang/rust#62127
 pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
@@ -261,12 +300,16 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
 ///
 /// Note that `start_paused` requires the `test-util` feature to be enabled.
 ///
-/// ### NOTE:
+/// ### Rename package
 ///
-/// If you rename the Tokio crate in your dependencies this macro will not work.
-/// If you must rename the current version of Tokio because you're also using an
-/// older version of Tokio, you _must_ make the current version of Tokio
-/// available as `tokio` in the module where this macro is expanded.
+/// ```rust
+/// use tokio as tokio1;
+///
+/// #[tokio1::test(crate = "tokio1")]
+/// async fn my_test() {
+///     println!("Hello world");
+/// }
+/// ```
 #[proc_macro_attribute]
 pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
     entry::test(args, item, true)
@@ -282,13 +325,6 @@ pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
 ///     assert!(true);
 /// }
 /// ```
-///
-/// ### NOTE:
-///
-/// If you rename the Tokio crate in your dependencies this macro will not work.
-/// If you must rename the current version of Tokio because you're also using an
-/// older version of Tokio, you _must_ make the current version of Tokio
-/// available as `tokio` in the module where this macro is expanded.
 #[proc_macro_attribute]
 pub fn test_rt(args: TokenStream, item: TokenStream) -> TokenStream {
     entry::test(args, item, false)
@@ -329,3 +365,11 @@ pub fn test_fail(_args: TokenStream, _item: TokenStream) -> TokenStream {
 pub fn select_priv_declare_output_enum(input: TokenStream) -> TokenStream {
     select::declare_output_enum(input)
 }
+
+/// Implementation detail of the `select!` macro. This macro is **not** intended
+/// to be used as part of the public API and is permitted to change.
+#[proc_macro]
+#[doc(hidden)]
+pub fn select_priv_clean_pattern(input: TokenStream) -> TokenStream {
+    select::clean_pattern_macro(input)
+}

tokio-macros/src/select.rs

@@ -41,3 +41,70 @@ pub(crate) fn declare_output_enum(input: TokenStream) -> TokenStream {
         pub(super) type Mask = #mask;
     })
 }
+
+pub(crate) fn clean_pattern_macro(input: TokenStream) -> TokenStream {
+    // If this isn't a pattern, we return the token stream as-is. The select!
+    // macro is using it in a location requiring a pattern, so an error will be
+    // emitted there.
+    let mut input: syn::Pat = match syn::parse(input.clone()) {
+        Ok(it) => it,
+        Err(_) => return input,
+    };
+
+    clean_pattern(&mut input);
+    quote::ToTokens::into_token_stream(input).into()
+}
+
+// Removes any occurrences of ref or mut in the provided pattern.
+fn clean_pattern(pat: &mut syn::Pat) {
+    match pat {
+        syn::Pat::Box(_box) => {}
+        syn::Pat::Lit(_literal) => {}
+        syn::Pat::Macro(_macro) => {}
+        syn::Pat::Path(_path) => {}
+        syn::Pat::Range(_range) => {}
+        syn::Pat::Rest(_rest) => {}
+        syn::Pat::Verbatim(_tokens) => {}
+        syn::Pat::Wild(_underscore) => {}
+        syn::Pat::Ident(ident) => {
+            ident.by_ref = None;
+            ident.mutability = None;
+            if let Some((_at, pat)) = &mut ident.subpat {
+                clean_pattern(&mut *pat);
+            }
+        }
+        syn::Pat::Or(or) => {
+            for case in or.cases.iter_mut() {
+                clean_pattern(case);
+            }
+        }
+        syn::Pat::Slice(slice) => {
+            for elem in slice.elems.iter_mut() {
+                clean_pattern(elem);
+            }
+        }
+        syn::Pat::Struct(struct_pat) => {
+            for field in struct_pat.fields.iter_mut() {
+                clean_pattern(&mut field.pat);
+            }
+        }
+        syn::Pat::Tuple(tuple) => {
+            for elem in tuple.elems.iter_mut() {
+                clean_pattern(elem);
+            }
+        }
+        syn::Pat::TupleStruct(tuple) => {
+            for elem in tuple.pat.elems.iter_mut() {
+                clean_pattern(elem);
+            }
+        }
+        syn::Pat::Reference(reference) => {
+            reference.mutability = None;
+            clean_pattern(&mut *reference.pat);
+        }
+        syn::Pat::Type(type_pat) => {
+            clean_pattern(&mut *type_pat.pat);
+        }
+        _ => {}
+    }
+}

tokio-stream/CHANGELOG.md

@@ -1,3 +1,33 @@
+# 0.1.9 (June 4, 2022)
+
+- deps: upgrade `tokio-util` dependency to `0.7.x` ([#3762])
+- stream: add `StreamExt::map_while` ([#4351])
+- stream: add `StreamExt::then` ([#4355])
+- stream: add cancel-safety docs to `StreamExt::next` and `try_next` ([#4715])
+- stream: expose `Elapsed` error ([#4502])
+- stream: expose `Timeout` ([#4601])
+- stream: implement `Extend` for `StreamMap` ([#4272])
+- sync: add `Clone` to `RecvError` types ([#4560])
+
+[#3762]: https://github.com/tokio-rs/tokio/pull/3762
+[#4272]: https://github.com/tokio-rs/tokio/pull/4272
+[#4351]: https://github.com/tokio-rs/tokio/pull/4351
+[#4355]: https://github.com/tokio-rs/tokio/pull/4355
+[#4502]: https://github.com/tokio-rs/tokio/pull/4502
+[#4560]: https://github.com/tokio-rs/tokio/pull/4560
+[#4601]: https://github.com/tokio-rs/tokio/pull/4601
+[#4715]: https://github.com/tokio-rs/tokio/pull/4715
+
+# 0.1.8 (October 29, 2021)
+
+- stream: add `From<Receiver<T>>` impl for receiver streams ([#4080])
+- stream: impl `FromIterator` for `StreamMap` ([#4052])
+- signal: make windows docs for signal module show up on unix builds ([#3770])
+
+[#3770]: https://github.com/tokio-rs/tokio/pull/3770
+[#4052]: https://github.com/tokio-rs/tokio/pull/4052
+[#4080]: https://github.com/tokio-rs/tokio/pull/4080
+
 # 0.1.7 (July 7, 2021)
 
 ### Fixed

tokio-stream/Cargo.toml

@@ -2,17 +2,15 @@
 name = "tokio-stream"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update doc url
-#   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-stream-0.1.x" git tag.
-version = "0.1.7"
+version = "0.1.9"
 edition = "2018"
+rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-stream/0.1.7/tokio_stream"
 description = """
 Utilities to work with `Stream` and `tokio`.
 """
@@ -31,7 +29,7 @@ signal = ["tokio/signal"]
 futures-core = { version = "0.3.0" }
 pin-project-lite = "0.2.0"
 tokio = { version = "1.8.0", path = "../tokio", features = ["sync"] }
-tokio-util = { version = "0.6.3", path = "../tokio-util", optional = true }
+tokio-util = { version = "0.7.0", path = "../tokio-util", optional = true }
 
 [dev-dependencies]
 tokio = { version = "1.2.0", path = "../tokio", features = ["full", "test-util"] }

tokio-stream/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-stream/src/lib.rs

@@ -10,7 +10,6 @@
     unreachable_pub
 )]
 #![cfg_attr(docsrs, feature(doc_cfg))]
-#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
@@ -78,6 +77,9 @@ pub mod wrappers;
 
 mod stream_ext;
 pub use stream_ext::{collect::FromStream, StreamExt};
+cfg_time! {
+    pub use stream_ext::timeout::{Elapsed, Timeout};
+}
 
 mod empty;
 pub use empty::{empty, Empty};

tokio-stream/src/stream_ext.rs

@@ -1,3 +1,4 @@
+use core::future::Future;
 use futures_core::Stream;
 
 mod all;
@@ -27,6 +28,9 @@ use fuse::Fuse;
 mod map;
 use map::Map;
 
+mod map_while;
+use map_while::MapWhile;
+
 mod merge;
 use merge::Merge;
 
@@ -39,17 +43,20 @@ use skip::Skip;
 mod skip_while;
 use skip_while::SkipWhile;
 
-mod try_next;
-use try_next::TryNext;
-
 mod take;
 use take::Take;
 
 mod take_while;
 use take_while::TakeWhile;
 
+mod then;
+use then::Then;
+
+mod try_next;
+use try_next::TryNext;
+
 cfg_time! {
-    mod timeout;
+    pub(crate) mod timeout;
     use timeout::Timeout;
     use tokio::time::Duration;
     mod throttle;
@@ -106,6 +113,12 @@ pub trait StreamExt: Stream {
     /// pinning it to the stack using the `pin_mut!` macro from the `pin_utils`
     /// crate.
     ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. The returned future only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     /// # Examples
     ///
     /// ```
@@ -142,6 +155,12 @@ pub trait StreamExt: Stream {
     /// an [`Option<Result<T, E>>`](Option), making for easy use
     /// with the [`?`](std::ops::Try) operator.
     ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. The returned future only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     /// # Examples
     ///
     /// ```
@@ -197,6 +216,93 @@ pub trait StreamExt: Stream {
         Map::new(self, f)
     }
 
+    /// Map this stream's items to a different type for as long as determined by
+    /// the provided closure. A stream of the target type will be returned,
+    /// which will yield elements until the closure returns `None`.
+    ///
+    /// The provided closure is executed over all elements of this stream as
+    /// they are made available, until it returns `None`. It is executed inline
+    /// with calls to [`poll_next`](Stream::poll_next). Once `None` is returned,
+    /// the underlying stream will not be polled again.
+    ///
+    /// Note that this function consumes the stream passed into it and returns a
+    /// wrapped version of it, similar to the [`Iterator::map_while`] method in the
+    /// standard library.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let stream = stream::iter(1..=10);
+    /// let mut stream = stream.map_while(|x| {
+    ///     if x < 4 {
+    ///         Some(x + 3)
+    ///     } else {
+    ///         None
+    ///     }
+    /// });
+    /// assert_eq!(stream.next().await, Some(4));
+    /// assert_eq!(stream.next().await, Some(5));
+    /// assert_eq!(stream.next().await, Some(6));
+    /// assert_eq!(stream.next().await, None);
+    /// # }
+    /// ```
+    fn map_while<T, F>(self, f: F) -> MapWhile<Self, F>
+    where
+        F: FnMut(Self::Item) -> Option<T>,
+        Self: Sized,
+    {
+        MapWhile::new(self, f)
+    }
+
+    /// Maps this stream's items asynchronously to a different type, returning a
+    /// new stream of the resulting type.
+    ///
+    /// The provided closure is executed over all elements of this stream as
+    /// they are made available, and the returned future is executed. Only one
+    /// future is executed at the time.
+    ///
+    /// Note that this function consumes the stream passed into it and returns a
+    /// wrapped version of it, similar to the existing `then` methods in the
+    /// standard library.
+    ///
+    /// Be aware that if the future is not `Unpin`, then neither is the `Stream`
+    /// returned by this method. To handle this, you can use `tokio::pin!` as in
+    /// the example below or put the stream in a `Box` with `Box::pin(stream)`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// async fn do_async_work(value: i32) -> i32 {
+    ///     value + 3
+    /// }
+    ///
+    /// let stream = stream::iter(1..=3);
+    /// let stream = stream.then(do_async_work);
+    ///
+    /// tokio::pin!(stream);
+    ///
+    /// assert_eq!(stream.next().await, Some(4));
+    /// assert_eq!(stream.next().await, Some(5));
+    /// assert_eq!(stream.next().await, Some(6));
+    /// # }
+    /// ```
+    fn then<F, Fut>(self, f: F) -> Then<Self, Fut, F>
+    where
+        F: FnMut(Self::Item) -> Fut,
+        Fut: Future,
+        Self: Sized,
+    {
+        Then::new(self, f)
+    }
+
     /// Combine two streams into one by interleaving the output of both as it
     /// is produced.
     ///

tokio-stream/src/stream_ext/collect.rs

@@ -66,17 +66,17 @@ where
         use Poll::Ready;
 
         loop {
-            let mut me = self.as_mut().project();
+            let me = self.as_mut().project();
 
             let item = match ready!(me.stream.poll_next(cx)) {
                 Some(item) => item,
                 None => {
-                    return Ready(U::finalize(sealed::Internal, &mut me.collection));
+                    return Ready(U::finalize(sealed::Internal, me.collection));
                 }
             };
 
-            if !U::extend(sealed::Internal, &mut me.collection, item) {
-                return Ready(U::finalize(sealed::Internal, &mut me.collection));
+            if !U::extend(sealed::Internal, me.collection, item) {
+                return Ready(U::finalize(sealed::Internal, me.collection));
             }
         }
     }

tokio-stream/src/stream_ext/map_while.rs

@@ -0,0 +1,52 @@
+use crate::Stream;
+
+use core::fmt;
+use core::pin::Pin;
+use core::task::{Context, Poll};
+use pin_project_lite::pin_project;
+
+pin_project! {
+    /// Stream for the [`map_while`](super::StreamExt::map_while) method.
+    #[must_use = "streams do nothing unless polled"]
+    pub struct MapWhile<St, F> {
+        #[pin]
+        stream: St,
+        f: F,
+    }
+}
+
+impl<St, F> fmt::Debug for MapWhile<St, F>
+where
+    St: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("MapWhile")
+            .field("stream", &self.stream)
+            .finish()
+    }
+}
+
+impl<St, F> MapWhile<St, F> {
+    pub(super) fn new(stream: St, f: F) -> Self {
+        MapWhile { stream, f }
+    }
+}
+
+impl<St, F, T> Stream for MapWhile<St, F>
+where
+    St: Stream,
+    F: FnMut(St::Item) -> Option<T>,
+{
+    type Item = T;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<T>> {
+        let me = self.project();
+        let f = me.f;
+        me.stream.poll_next(cx).map(|opt| opt.and_then(f))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (_, upper) = self.stream.size_hint();
+        (0, upper)
+    }
+}

tokio-stream/src/stream_ext/next.rs

@@ -8,6 +8,13 @@ use pin_project_lite::pin_project;
 
 pin_project! {
     /// Future for the [`next`](super::StreamExt::next) method.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. It only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
+    ///
     #[derive(Debug)]
     #[must_use = "futures do nothing unless you `.await` or poll them"]
     pub struct Next<'a, St: ?Sized> {

tokio-stream/src/stream_ext/then.rs

@@ -0,0 +1,83 @@
+use crate::Stream;
+
+use core::fmt;
+use core::future::Future;
+use core::pin::Pin;
+use core::task::{Context, Poll};
+use pin_project_lite::pin_project;
+
+pin_project! {
+    /// Stream for the [`then`](super::StreamExt::then) method.
+    #[must_use = "streams do nothing unless polled"]
+    pub struct Then<St, Fut, F> {
+        #[pin]
+        stream: St,
+        #[pin]
+        future: Option<Fut>,
+        f: F,
+    }
+}
+
+impl<St, Fut, F> fmt::Debug for Then<St, Fut, F>
+where
+    St: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Then")
+            .field("stream", &self.stream)
+            .finish()
+    }
+}
+
+impl<St, Fut, F> Then<St, Fut, F> {
+    pub(super) fn new(stream: St, f: F) -> Self {
+        Then {
+            stream,
+            future: None,
+            f,
+        }
+    }
+}
+
+impl<St, F, Fut> Stream for Then<St, Fut, F>
+where
+    St: Stream,
+    Fut: Future,
+    F: FnMut(St::Item) -> Fut,
+{
+    type Item = Fut::Output;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Fut::Output>> {
+        let mut me = self.project();
+
+        loop {
+            if let Some(future) = me.future.as_mut().as_pin_mut() {
+                match future.poll(cx) {
+                    Poll::Ready(item) => {
+                        me.future.set(None);
+                        return Poll::Ready(Some(item));
+                    }
+                    Poll::Pending => return Poll::Pending,
+                }
+            }
+
+            match me.stream.as_mut().poll_next(cx) {
+                Poll::Ready(Some(item)) => {
+                    me.future.set(Some((me.f)(item)));
+                }
+                Poll::Ready(None) => return Poll::Ready(None),
+                Poll::Pending => return Poll::Pending,
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let future_len = if self.future.is_some() { 1 } else { 0 };
+        let (lower, upper) = self.stream.size_hint();
+
+        let lower = lower.saturating_add(future_len);
+        let upper = upper.and_then(|upper| upper.checked_add(future_len));
+
+        (lower, upper)
+    }
+}

tokio-stream/src/stream_ext/try_next.rs

@@ -9,6 +9,12 @@ use pin_project_lite::pin_project;
 
 pin_project! {
     /// Future for the [`try_next`](super::StreamExt::try_next) method.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe. It only
+    /// holds onto a reference to the underlying stream,
+    /// so dropping it will never lose a value.
     #[derive(Debug)]
     #[must_use = "futures do nothing unless you `.await` or poll them"]
     pub struct TryNext<'a, St: ?Sized> {

tokio-stream/src/stream_map.rs

@@ -585,6 +585,15 @@ where
     }
 }
 
+impl<K, V> Extend<(K, V)> for StreamMap<K, V> {
+    fn extend<T>(&mut self, iter: T)
+    where
+        T: IntoIterator<Item = (K, V)>,
+    {
+        self.entries.extend(iter);
+    }
+}
+
 mod rand {
     use std::cell::Cell;
 

tokio-stream/src/wrappers/broadcast.rs

@@ -14,11 +14,11 @@ use std::task::{Context, Poll};
 /// [`Stream`]: trait@crate::Stream
 #[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
 pub struct BroadcastStream<T> {
-    inner: ReusableBoxFuture<(Result<T, RecvError>, Receiver<T>)>,
+    inner: ReusableBoxFuture<'static, (Result<T, RecvError>, Receiver<T>)>,
 }
 
 /// An error returned from the inner stream of a [`BroadcastStream`].
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum BroadcastStreamRecvError {
     /// The receiver lagged too far behind. Attempting to receive again will
     /// return the oldest message still retained by the channel.

tokio-stream/src/wrappers/watch.rs

@@ -49,7 +49,7 @@ use tokio::sync::watch::error::RecvError;
 /// [`Stream`]: trait@crate::Stream
 #[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
 pub struct WatchStream<T> {
-    inner: ReusableBoxFuture<(Result<(), RecvError>, Receiver<T>)>,
+    inner: ReusableBoxFuture<'static, (Result<(), RecvError>, Receiver<T>)>,
 }
 
 async fn make_future<T: Clone + Send + Sync>(

tokio-stream/tests/stream_timeout.rs

@@ -1,10 +1,10 @@
-#![cfg(feature = "full")]
+#![cfg(all(feature = "time", feature = "sync", feature = "io-util"))]
 
 use tokio::time::{self, sleep, Duration};
 use tokio_stream::{self, StreamExt};
 use tokio_test::*;
 
-use futures::StreamExt as _;
+use futures::stream;
 
 async fn maybe_sleep(idx: i32) -> i32 {
     if idx % 2 == 0 {

tokio-stream/tests/time_throttle.rs

@@ -1,5 +1,5 @@
 #![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "time", feature = "sync", feature = "io-util"))]
 
 use tokio::time;
 use tokio_stream::StreamExt;

tokio-test/Cargo.toml

@@ -2,17 +2,15 @@
 name = "tokio-test"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update doc url
-#   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-test-0.4.x" git tag.
 version = "0.4.2"
 edition = "2018"
+rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-test/0.4.2/tokio_test"
 description = """
 Testing utilities for Tokio- and futures-based code
 """
@@ -20,7 +18,7 @@ categories = ["asynchronous", "testing"]
 
 [dependencies]
 tokio = { version = "1.2.0", path = "../tokio", features = ["rt", "sync", "time", "test-util"] }
-tokio-stream = { version = "0.1", path = "../tokio-stream" }
+tokio-stream = { version = "0.1.1", path = "../tokio-stream" }
 async-stream = "0.3"
 
 bytes = "1.0.0"

tokio-test/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-test/src/lib.rs

@@ -4,7 +4,6 @@
     rust_2018_idioms,
     unreachable_pub
 )]
-#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))

tokio-util/CHANGELOG.md

@@ -1,3 +1,82 @@
+# 0.7.3 (June 4, 2022)
+
+### Changed
+
+- tracing: don't require default tracing features ([#4592])
+- util: simplify implementation of `ReusableBoxFuture` ([#4675])
+
+### Added (unstable)
+
+- task: add `JoinMap` ([#4640], [#4697])
+
+[#4592]: https://github.com/tokio-rs/tokio/pull/4592
+[#4640]: https://github.com/tokio-rs/tokio/pull/4640
+[#4675]: https://github.com/tokio-rs/tokio/pull/4675
+[#4697]: https://github.com/tokio-rs/tokio/pull/4697
+
+# 0.7.2 (May 14, 2022)
+
+This release contains a rewrite of `CancellationToken` that fixes a memory leak. ([#4652])
+
+[#4652]: https://github.com/tokio-rs/tokio/pull/4652
+
+# 0.7.1 (February 21, 2022)
+
+### Added
+
+- codec: add `length_field_type` to `LengthDelimitedCodec` builder ([#4508])
+- io: add `StreamReader::into_inner_with_chunk()` ([#4559])
+
+### Changed
+
+- switch from log to tracing ([#4539])
+
+### Fixed
+
+- sync: fix waker update condition in `CancellationToken` ([#4497])
+- bumped tokio dependency to 1.6 to satisfy minimum requirements ([#4490])
+
+[#4490]: https://github.com/tokio-rs/tokio/pull/4490
+[#4497]: https://github.com/tokio-rs/tokio/pull/4497
+[#4508]: https://github.com/tokio-rs/tokio/pull/4508
+[#4539]: https://github.com/tokio-rs/tokio/pull/4539
+[#4559]: https://github.com/tokio-rs/tokio/pull/4559
+
+# 0.7.0 (February 9, 2022)
+
+### Added
+
+- task: add `spawn_pinned` ([#3370])
+- time: add `shrink_to_fit` and `compact` methods to `DelayQueue` ([#4170])
+- codec: improve `Builder::max_frame_length` docs ([#4352])
+- codec: add mutable reference getters for codecs to pinned `Framed` ([#4372])
+- net: add generic trait to combine `UnixListener` and `TcpListener` ([#4385])
+- codec: implement `Framed::map_codec` ([#4427])
+- codec: implement `Encoder<BytesMut>` for `BytesCodec` ([#4465])
+
+### Changed
+
+- sync: add lifetime parameter to `ReusableBoxFuture` ([#3762])
+- sync: refactored `PollSender<T>` to fix a subtly broken `Sink<T>` implementation ([#4214])
+- time: remove error case from the infallible `DelayQueue::poll_elapsed` ([#4241])
+
+[#3370]: https://github.com/tokio-rs/tokio/pull/3370
+[#4170]: https://github.com/tokio-rs/tokio/pull/4170
+[#4352]: https://github.com/tokio-rs/tokio/pull/4352
+[#4372]: https://github.com/tokio-rs/tokio/pull/4372
+[#4385]: https://github.com/tokio-rs/tokio/pull/4385
+[#4427]: https://github.com/tokio-rs/tokio/pull/4427
+[#4465]: https://github.com/tokio-rs/tokio/pull/4465
+[#3762]: https://github.com/tokio-rs/tokio/pull/3762
+[#4214]: https://github.com/tokio-rs/tokio/pull/4214
+[#4241]: https://github.com/tokio-rs/tokio/pull/4241
+
+# 0.6.10 (May 14, 2021)
+
+This is a backport for the memory leak in `CancellationToken` that was originally fixed in 0.7.2. ([#4652])
+
+[#4652]: https://github.com/tokio-rs/tokio/pull/4652
+
 # 0.6.9 (October 29, 2021)
 
 ### Added

tokio-util/Cargo.toml

@@ -2,17 +2,15 @@
 name = "tokio-util"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update doc url
-#   - Cargo.toml
 # - Update CHANGELOG.md.
-# - Create "tokio-util-0.6.x" git tag.
-version = "0.6.9"
+# - Create "tokio-util-0.7.x" git tag.
+version = "0.7.3"
 edition = "2018"
+rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-util/0.6.9/tokio_util"
 description = """
 Additional utilities for working with Tokio.
 """
@@ -27,25 +25,27 @@ full = ["codec", "compat", "io-util", "time", "net", "rt"]
 
 net = ["tokio/net"]
 compat = ["futures-io",]
-codec = []
+codec = ["tracing"]
 time = ["tokio/time","slab"]
 io = []
 io-util = ["io", "tokio/rt", "tokio/io-util"]
-rt = ["tokio/rt"]
+rt = ["tokio/rt", "tokio/sync", "futures-util", "hashbrown"]
 
 __docs_rs = ["futures-util"]
 
 [dependencies]
-tokio = { version = "1.0.0", path = "../tokio", features = ["sync"] }
-
+tokio = { version = "1.19.0", path = "../tokio", features = ["sync"] }
 bytes = "1.0.0"
 futures-core = "0.3.0"
 futures-sink = "0.3.0"
 futures-io = { version = "0.3.0", optional = true }
 futures-util = { version = "0.3.0", optional = true }
-log = "0.4"
 pin-project-lite = "0.2.0"
-slab = { version = "0.4.1", optional = true } # Backs `DelayQueue`
+slab = { version = "0.4.4", optional = true } # Backs `DelayQueue`
+tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true }
+
+[target.'cfg(tokio_unstable)'.dependencies]
+hashbrown = { version = "0.12.0", optional = true }
 
 [dev-dependencies]
 tokio = { version = "1.0.0", path = "../tokio", features = ["full"] }
@@ -58,4 +58,8 @@ futures-test = "0.3.5"
 
 [package.metadata.docs.rs]
 all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
+# enable unstable features in the documentation
+rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]
+# it's necessary to _also_ pass `--cfg tokio_unstable` to rustc, or else
+# dependencies will not be enabled, and the docs build will fail.
+rustc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]

tokio-util/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-util/src/codec/bytes_codec.rs

@@ -74,3 +74,13 @@ impl Encoder<Bytes> for BytesCodec {
         Ok(())
     }
 }
+
+impl Encoder<BytesMut> for BytesCodec {
+    type Error = io::Error;
+
+    fn encode(&mut self, data: BytesMut, buf: &mut BytesMut) -> Result<(), io::Error> {
+        buf.reserve(data.len());
+        buf.put(data);
+        Ok(())
+    }
+}

tokio-util/src/codec/framed.rs

@@ -204,6 +204,35 @@ impl<T, U> Framed<T, U> {
         &mut self.inner.codec
     }
 
+    /// Maps the codec `U` to `C`, preserving the read and write buffers
+    /// wrapped by `Framed`.
+    ///
+    /// Note that care should be taken to not tamper with the underlying codec
+    /// as it may corrupt the stream of frames otherwise being worked with.
+    pub fn map_codec<C, F>(self, map: F) -> Framed<T, C>
+    where
+        F: FnOnce(U) -> C,
+    {
+        // This could be potentially simplified once rust-lang/rust#86555 hits stable
+        let parts = self.into_parts();
+        Framed::from_parts(FramedParts {
+            io: parts.io,
+            codec: map(parts.codec),
+            read_buf: parts.read_buf,
+            write_buf: parts.write_buf,
+            _priv: (),
+        })
+    }
+
+    /// Returns a mutable reference to the underlying codec wrapped by
+    /// `Framed`.
+    ///
+    /// Note that care should be taken to not tamper with the underlying codec
+    /// as it may corrupt the stream of frames otherwise being worked with.
+    pub fn codec_pin_mut(self: Pin<&mut Self>) -> &mut U {
+        self.project().inner.project().codec
+    }
+
     /// Returns a reference to the read buffer.
     pub fn read_buffer(&self) -> &BytesMut {
         &self.inner.state.read.buffer

tokio-util/src/codec/framed_impl.rs

@@ -7,12 +7,12 @@ use tokio::io::{AsyncRead, AsyncWrite};
 use bytes::BytesMut;
 use futures_core::ready;
 use futures_sink::Sink;
-use log::trace;
 use pin_project_lite::pin_project;
 use std::borrow::{Borrow, BorrowMut};
 use std::io;
 use std::pin::Pin;
 use std::task::{Context, Poll};
+use tracing::trace;
 
 pin_project! {
     #[derive(Debug)]
@@ -278,7 +278,7 @@ where
 
         while !pinned.state.borrow_mut().buffer.is_empty() {
             let WriteFrame { buffer } = pinned.state.borrow_mut();
-            trace!("writing; remaining={}", buffer.len());
+            trace!(remaining = buffer.len(), "writing;");
 
             let n = ready!(poll_write_buf(pinned.inner.as_mut(), cx, buffer))?;
 

tokio-util/src/codec/framed_read.rs

@@ -108,6 +108,32 @@ impl<T, D> FramedRead<T, D> {
         &mut self.inner.codec
     }
 
+    /// Maps the decoder `D` to `C`, preserving the read buffer
+    /// wrapped by `Framed`.
+    pub fn map_decoder<C, F>(self, map: F) -> FramedRead<T, C>
+    where
+        F: FnOnce(D) -> C,
+    {
+        // This could be potentially simplified once rust-lang/rust#86555 hits stable
+        let FramedImpl {
+            inner,
+            state,
+            codec,
+        } = self.inner;
+        FramedRead {
+            inner: FramedImpl {
+                inner,
+                state,
+                codec: map(codec),
+            },
+        }
+    }
+
+    /// Returns a mutable reference to the underlying decoder.
+    pub fn decoder_pin_mut(self: Pin<&mut Self>) -> &mut D {
+        self.project().inner.project().codec
+    }
+
     /// Returns a reference to the read buffer.
     pub fn read_buffer(&self) -> &BytesMut {
         &self.inner.state.buffer

tokio-util/src/codec/framed_write.rs

@@ -88,6 +88,32 @@ impl<T, E> FramedWrite<T, E> {
         &mut self.inner.codec
     }
 
+    /// Maps the encoder `E` to `C`, preserving the write buffer
+    /// wrapped by `Framed`.
+    pub fn map_encoder<C, F>(self, map: F) -> FramedWrite<T, C>
+    where
+        F: FnOnce(E) -> C,
+    {
+        // This could be potentially simplified once rust-lang/rust#86555 hits stable
+        let FramedImpl {
+            inner,
+            state,
+            codec,
+        } = self.inner;
+        FramedWrite {
+            inner: FramedImpl {
+                inner,
+                state,
+                codec: map(codec),
+            },
+        }
+    }
+
+    /// Returns a mutable reference to the underlying encoder.
+    pub fn encoder_pin_mut(self: Pin<&mut Self>) -> &mut E {
+        self.project().inner.project().codec
+    }
+
     /// Returns a reference to the write buffer.
     pub fn write_buffer(&self) -> &BytesMut {
         &self.inner.state.buffer

tokio-util/src/codec/length_delimited.rs

@@ -84,7 +84,7 @@
 //! # fn bind_read<T: AsyncRead>(io: T) {
 //! LengthDelimitedCodec::builder()
 //!     .length_field_offset(0) // default value
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .length_adjustment(0)   // default value
 //!     .num_skip(0) // Do not strip frame header
 //!     .new_read(io);
@@ -118,7 +118,7 @@
 //! # fn bind_read<T: AsyncRead>(io: T) {
 //! LengthDelimitedCodec::builder()
 //!     .length_field_offset(0) // default value
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .length_adjustment(0)   // default value
 //!     // `num_skip` is not needed, the default is to skip
 //!     .new_read(io);
@@ -150,7 +150,7 @@
 //! # fn bind_read<T: AsyncRead>(io: T) {
 //! LengthDelimitedCodec::builder()
 //!     .length_field_offset(0) // default value
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .length_adjustment(-2)  // size of head
 //!     .num_skip(0)
 //!     .new_read(io);
@@ -228,7 +228,7 @@
 //! # fn bind_read<T: AsyncRead>(io: T) {
 //! LengthDelimitedCodec::builder()
 //!     .length_field_offset(1) // length of hdr1
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .length_adjustment(1)  // length of hdr2
 //!     .num_skip(3) // length of hdr1 + LEN
 //!     .new_read(io);
@@ -274,7 +274,7 @@
 //! # fn bind_read<T: AsyncRead>(io: T) {
 //! LengthDelimitedCodec::builder()
 //!     .length_field_offset(1) // length of hdr1
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .length_adjustment(-3)  // length of hdr1 + LEN, negative
 //!     .num_skip(3)
 //!     .new_read(io);
@@ -350,7 +350,7 @@
 //! # fn write_frame<T: AsyncWrite>(io: T) {
 //! # let _ =
 //! LengthDelimitedCodec::builder()
-//!     .length_field_length(2)
+//!     .length_field_type::<u16>()
 //!     .new_write(io);
 //! # }
 //! # pub fn main() {}
@@ -379,7 +379,7 @@ use tokio::io::{AsyncRead, AsyncWrite};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use std::error::Error as StdError;
 use std::io::{self, Cursor};
-use std::{cmp, fmt};
+use std::{cmp, fmt, mem};
 
 /// Configure length delimited `LengthDelimitedCodec`s.
 ///
@@ -629,6 +629,24 @@ impl Default for LengthDelimitedCodec {
 
 // ===== impl Builder =====
 
+mod builder {
+    /// Types that can be used with `Builder::length_field_type`.
+    pub trait LengthFieldType {}
+
+    impl LengthFieldType for u8 {}
+    impl LengthFieldType for u16 {}
+    impl LengthFieldType for u32 {}
+    impl LengthFieldType for u64 {}
+
+    #[cfg(any(
+        target_pointer_width = "8",
+        target_pointer_width = "16",
+        target_pointer_width = "32",
+        target_pointer_width = "64",
+    ))]
+    impl LengthFieldType for usize {}
+}
+
 impl Builder {
     /// Creates a new length delimited codec builder with default configuration
     /// values.
@@ -642,7 +660,7 @@ impl Builder {
     /// # fn bind_read<T: AsyncRead>(io: T) {
     /// LengthDelimitedCodec::builder()
     ///     .length_field_offset(0)
-    ///     .length_field_length(2)
+    ///     .length_field_type::<u16>()
     ///     .length_adjustment(0)
     ///     .num_skip(0)
     ///     .new_read(io);
@@ -746,7 +764,7 @@ impl Builder {
         }
     }
 
-    /// Sets the max frame length
+    /// Sets the max frame length in bytes
     ///
     /// This configuration option applies to both encoding and decoding. The
     /// default value is 8MB.
@@ -767,7 +785,7 @@ impl Builder {
     ///
     /// # fn bind_read<T: AsyncRead>(io: T) {
     /// LengthDelimitedCodec::builder()
-    ///     .max_frame_length(8 * 1024)
+    ///     .max_frame_length(8 * 1024 * 1024)
     ///     .new_read(io);
     /// # }
     /// # pub fn main() {}
@@ -777,6 +795,42 @@ impl Builder {
         self
     }
 
+    /// Sets the unsigned integer type used to represent the length field.
+    ///
+    /// The default type is [`u32`]. The max type is [`u64`] (or [`usize`] on
+    /// 64-bit targets).
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use tokio::io::AsyncRead;
+    /// use tokio_util::codec::LengthDelimitedCodec;
+    ///
+    /// # fn bind_read<T: AsyncRead>(io: T) {
+    /// LengthDelimitedCodec::builder()
+    ///     .length_field_type::<u32>()
+    ///     .new_read(io);
+    /// # }
+    /// # pub fn main() {}
+    /// ```
+    ///
+    /// Unlike [`Builder::length_field_length`], this does not fail at runtime
+    /// and instead produces a compile error:
+    ///
+    /// ```compile_fail
+    /// # use tokio::io::AsyncRead;
+    /// # use tokio_util::codec::LengthDelimitedCodec;
+    /// # fn bind_read<T: AsyncRead>(io: T) {
+    /// LengthDelimitedCodec::builder()
+    ///     .length_field_type::<u128>()
+    ///     .new_read(io);
+    /// # }
+    /// # pub fn main() {}
+    /// ```
+    pub fn length_field_type<T: builder::LengthFieldType>(&mut self) -> &mut Self {
+        self.length_field_length(mem::size_of::<T>())
+    }
+
     /// Sets the number of bytes used to represent the length field
     ///
     /// The default value is `4`. The max value is `8`.
@@ -878,7 +932,7 @@ impl Builder {
     /// # pub fn main() {
     /// LengthDelimitedCodec::builder()
     ///     .length_field_offset(0)
-    ///     .length_field_length(2)
+    ///     .length_field_type::<u16>()
     ///     .length_adjustment(0)
     ///     .num_skip(0)
     ///     .new_codec();
@@ -902,7 +956,7 @@ impl Builder {
     /// # fn bind_read<T: AsyncRead>(io: T) {
     /// LengthDelimitedCodec::builder()
     ///     .length_field_offset(0)
-    ///     .length_field_length(2)
+    ///     .length_field_type::<u16>()
     ///     .length_adjustment(0)
     ///     .num_skip(0)
     ///     .new_read(io);
@@ -925,7 +979,7 @@ impl Builder {
     /// # use tokio_util::codec::LengthDelimitedCodec;
     /// # fn write_frame<T: AsyncWrite>(io: T) {
     /// LengthDelimitedCodec::builder()
-    ///     .length_field_length(2)
+    ///     .length_field_type::<u16>()
     ///     .new_write(io);
     /// # }
     /// # pub fn main() {}
@@ -947,7 +1001,7 @@ impl Builder {
     /// # fn write_frame<T: AsyncRead + AsyncWrite>(io: T) {
     /// # let _ =
     /// LengthDelimitedCodec::builder()
-    ///     .length_field_length(2)
+    ///     .length_field_type::<u16>()
     ///     .new_framed(io);
     /// # }
     /// # pub fn main() {}

tokio-util/src/io/stream_reader.rs

@@ -84,13 +84,24 @@ where
     }
 
     /// Do we have a chunk and is it non-empty?
-    fn has_chunk(self: Pin<&mut Self>) -> bool {
-        if let Some(chunk) = self.project().chunk {
+    fn has_chunk(&self) -> bool {
+        if let Some(ref chunk) = self.chunk {
             chunk.remaining() > 0
         } else {
             false
         }
     }
+
+    /// Consumes this `StreamReader`, returning a Tuple consisting
+    /// of the underlying stream and an Option of the interal buffer,
+    /// which is Some in case the buffer contains elements.
+    pub fn into_inner_with_chunk(self) -> (S, Option<B>) {
+        if self.has_chunk() {
+            (self.inner, self.chunk)
+        } else {
+            (self.inner, None)
+        }
+    }
 }
 
 impl<S, B> StreamReader<S, B> {
@@ -118,6 +129,10 @@ impl<S, B> StreamReader<S, B> {
     /// Consumes this `BufWriter`, returning the underlying stream.
     ///
     /// Note that any leftover data in the internal buffer is lost.
+    /// If you additionally want access to the internal buffer use
+    /// [`into_inner_with_chunk`].
+    ///
+    /// [`into_inner_with_chunk`]: crate::io::StreamReader::into_inner_with_chunk
     pub fn into_inner(self) -> S {
         self.inner
     }

tokio-util/src/lib.rs

@@ -5,7 +5,6 @@
     rust_2018_idioms,
     unreachable_pub
 )]
-#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
@@ -31,6 +30,7 @@ cfg_codec! {
 
 cfg_net! {
     pub mod udp;
+    pub mod net;
 }
 
 cfg_compat! {
@@ -43,6 +43,7 @@ cfg_io! {
 
 cfg_rt! {
     pub mod context;
+    pub mod task;
 }
 
 cfg_time! {
@@ -114,6 +115,9 @@ mod util {
 
         let n = {
             let dst = buf.chunk_mut();
+
+            // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
+            // transparent wrapper around `[MaybeUninit<u8>]`.
             let dst = unsafe { &mut *(dst as *mut _ as *mut [MaybeUninit<u8>]) };
             let mut buf = ReadBuf::uninit(dst);
             let ptr = buf.filled().as_ptr();

tokio-util/src/net/mod.rs

@@ -0,0 +1,97 @@
+//! TCP/UDP/Unix helpers for tokio.
+
+use crate::either::Either;
+use std::future::Future;
+use std::io::Result;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+#[cfg(unix)]
+pub mod unix;
+
+/// A trait for a listener: `TcpListener` and `UnixListener`.
+pub trait Listener {
+    /// The stream's type of this listener.
+    type Io: tokio::io::AsyncRead + tokio::io::AsyncWrite;
+    /// The socket address type of this listener.
+    type Addr;
+
+    /// Polls to accept a new incoming connection to this listener.
+    fn poll_accept(&mut self, cx: &mut Context<'_>) -> Poll<Result<(Self::Io, Self::Addr)>>;
+
+    /// Accepts a new incoming connection from this listener.
+    fn accept(&mut self) -> ListenerAcceptFut<'_, Self>
+    where
+        Self: Sized,
+    {
+        ListenerAcceptFut { listener: self }
+    }
+
+    /// Returns the local address that this listener is bound to.
+    fn local_addr(&self) -> Result<Self::Addr>;
+}
+
+impl Listener for tokio::net::TcpListener {
+    type Io = tokio::net::TcpStream;
+    type Addr = std::net::SocketAddr;
+
+    fn poll_accept(&mut self, cx: &mut Context<'_>) -> Poll<Result<(Self::Io, Self::Addr)>> {
+        Self::poll_accept(self, cx)
+    }
+
+    fn local_addr(&self) -> Result<Self::Addr> {
+        self.local_addr().map(Into::into)
+    }
+}
+
+/// Future for accepting a new connection from a listener.
+#[derive(Debug)]
+#[must_use = "futures do nothing unless you `.await` or poll them"]
+pub struct ListenerAcceptFut<'a, L> {
+    listener: &'a mut L,
+}
+
+impl<'a, L> Future for ListenerAcceptFut<'a, L>
+where
+    L: Listener,
+{
+    type Output = Result<(L::Io, L::Addr)>;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        self.listener.poll_accept(cx)
+    }
+}
+
+impl<L, R> Either<L, R>
+where
+    L: Listener,
+    R: Listener,
+{
+    /// Accepts a new incoming connection from this listener.
+    pub async fn accept(&mut self) -> Result<Either<(L::Io, L::Addr), (R::Io, R::Addr)>> {
+        match self {
+            Either::Left(listener) => {
+                let (stream, addr) = listener.accept().await?;
+                Ok(Either::Left((stream, addr)))
+            }
+            Either::Right(listener) => {
+                let (stream, addr) = listener.accept().await?;
+                Ok(Either::Right((stream, addr)))
+            }
+        }
+    }
+
+    /// Returns the local address that this listener is bound to.
+    pub fn local_addr(&self) -> Result<Either<L::Addr, R::Addr>> {
+        match self {
+            Either::Left(listener) => {
+                let addr = listener.local_addr()?;
+                Ok(Either::Left(addr))
+            }
+            Either::Right(listener) => {
+                let addr = listener.local_addr()?;
+                Ok(Either::Right(addr))
+            }
+        }
+    }
+}

tokio-util/src/net/unix/mod.rs

@@ -0,0 +1,18 @@
+//! Unix domain socket helpers.
+
+use super::Listener;
+use std::io::Result;
+use std::task::{Context, Poll};
+
+impl Listener for tokio::net::UnixListener {
+    type Io = tokio::net::UnixStream;
+    type Addr = tokio::net::unix::SocketAddr;
+
+    fn poll_accept(&mut self, cx: &mut Context<'_>) -> Poll<Result<(Self::Io, Self::Addr)>> {
+        Self::poll_accept(self, cx)
+    }
+
+    fn local_addr(&self) -> Result<Self::Addr> {
+        self.local_addr().map(Into::into)
+    }
+}

tokio-util/src/sync/cancellation_token.rs

@@ -1,18 +1,15 @@
 //! An asynchronously awaitable `CancellationToken`.
 //! The token allows to signal a cancellation request to one or more tasks.
 pub(crate) mod guard;
+mod tree_node;
 
-use crate::loom::sync::atomic::AtomicUsize;
-use crate::loom::sync::Mutex;
-use crate::sync::intrusive_double_linked_list::{LinkedList, ListNode};
-
+use crate::loom::sync::Arc;
 use core::future::Future;
 use core::pin::Pin;
-use core::ptr::NonNull;
-use core::sync::atomic::Ordering;
-use core::task::{Context, Poll, Waker};
+use core::task::{Context, Poll};
 
 use guard::DropGuard;
+use pin_project_lite::pin_project;
 
 /// A token which can be used to signal a cancellation request to one or more
 /// tasks.
@@ -24,9 +21,9 @@ use guard::DropGuard;
 ///
 /// # Examples
 ///
-/// ```ignore
+/// ```no_run
 /// use tokio::select;
-/// use tokio::scope::CancellationToken;
+/// use tokio_util::sync::CancellationToken;
 ///
 /// #[tokio::main]
 /// async fn main() {
@@ -55,31 +52,20 @@ use guard::DropGuard;
 /// }
 /// ```
 pub struct CancellationToken {
-    inner: NonNull<CancellationTokenState>,
+    inner: Arc<tree_node::TreeNode>,
 }
 
-// Safety: The CancellationToken is thread-safe and can be moved between threads,
-// since all methods are internally synchronized.
-unsafe impl Send for CancellationToken {}
-unsafe impl Sync for CancellationToken {}
-
-/// A Future that is resolved once the corresponding [`CancellationToken`]
-/// was cancelled
-#[must_use = "futures do nothing unless polled"]
-pub struct WaitForCancellationFuture<'a> {
-    /// The CancellationToken that is associated with this WaitForCancellationFuture
-    cancellation_token: Option<&'a CancellationToken>,
-    /// Node for waiting at the cancellation_token
-    wait_node: ListNode<WaitQueueEntry>,
-    /// Whether this future was registered at the token yet as a waiter
-    is_registered: bool,
+pin_project! {
+    /// A Future that is resolved once the corresponding [`CancellationToken`]
+    /// is cancelled.
+    #[must_use = "futures do nothing unless polled"]
+    pub struct WaitForCancellationFuture<'a> {
+        cancellation_token: &'a CancellationToken,
+        #[pin]
+        future: tokio::sync::futures::Notified<'a>,
+    }
 }
 
-// Safety: Futures can be sent between threads as long as the underlying
-// cancellation_token is thread-safe (Sync),
-// which allows to poll/register/unregister from a different thread.
-unsafe impl<'a> Send for WaitForCancellationFuture<'a> {}
-
 // ===== impl CancellationToken =====
 
 impl core::fmt::Debug for CancellationToken {
@@ -92,43 +78,16 @@ impl core::fmt::Debug for CancellationToken {
 
 impl Clone for CancellationToken {
     fn clone(&self) -> Self {
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        let inner = self.state();
-
-        // Tokens are cloned by increasing their refcount
-        let current_state = inner.snapshot();
-        inner.increment_refcount(current_state);
-
-        CancellationToken { inner: self.inner }
+        tree_node::increase_handle_refcount(&self.inner);
+        CancellationToken {
+            inner: self.inner.clone(),
+        }
     }
 }
 
 impl Drop for CancellationToken {
     fn drop(&mut self) {
-        let token_state_pointer = self.inner;
-
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        let inner = unsafe { &mut *self.inner.as_ptr() };
-
-        let mut current_state = inner.snapshot();
-
-        // We need to safe the parent, since the state might be released by the
-        // next call
-        let parent = inner.parent;
-
-        // Drop our own refcount
-        current_state = inner.decrement_refcount(current_state);
-
-        // If this was the last reference, unregister from the parent
-        if current_state.refcount == 0 {
-            if let Some(mut parent) = parent {
-                // Safety: Since we still retain a reference on the parent, it must be valid.
-                let parent = unsafe { parent.as_mut() };
-                parent.unregister_child(token_state_pointer, current_state);
-            }
-        }
+        tree_node::decrease_handle_refcount(&self.inner);
     }
 }
 
@@ -141,29 +100,11 @@ impl Default for CancellationToken {
 impl CancellationToken {
     /// Creates a new CancellationToken in the non-cancelled state.
     pub fn new() -> CancellationToken {
-        let state = Box::new(CancellationTokenState::new(
-            None,
-            StateSnapshot {
-                cancel_state: CancellationState::NotCancelled,
-                has_parent_ref: false,
-                refcount: 1,
-            },
-        ));
-
-        // Safety: We just created the Box. The pointer is guaranteed to be
-        // not null
         CancellationToken {
-            inner: unsafe { NonNull::new_unchecked(Box::into_raw(state)) },
+            inner: Arc::new(tree_node::TreeNode::new()),
         }
     }
 
-    /// Returns a reference to the utilized `CancellationTokenState`.
-    fn state(&self) -> &CancellationTokenState {
-        // Safety: The state inside a `CancellationToken` is always valid, since
-        // is reference counted
-        unsafe { &*self.inner.as_ptr() }
-    }
-
     /// Creates a `CancellationToken` which will get cancelled whenever the
     /// current token gets cancelled.
     ///
@@ -172,9 +113,9 @@ impl CancellationToken {
     ///
     /// # Examples
     ///
-    /// ```ignore
+    /// ```no_run
     /// use tokio::select;
-    /// use tokio::scope::CancellationToken;
+    /// use tokio_util::sync::CancellationToken;
     ///
     /// #[tokio::main]
     /// async fn main() {
@@ -203,56 +144,8 @@ impl CancellationToken {
     /// }
     /// ```
     pub fn child_token(&self) -> CancellationToken {
-        let inner = self.state();
-
-        // Increment the refcount of this token. It will be referenced by the
-        // child, independent of whether the child is immediately cancelled or
-        // not.
-        let _current_state = inner.increment_refcount(inner.snapshot());
-
-        let mut unpacked_child_state = StateSnapshot {
-            has_parent_ref: true,
-            refcount: 1,
-            cancel_state: CancellationState::NotCancelled,
-        };
-        let mut child_token_state = Box::new(CancellationTokenState::new(
-            Some(self.inner),
-            unpacked_child_state,
-        ));
-
-        {
-            let mut guard = inner.synchronized.lock().unwrap();
-            if guard.is_cancelled {
-                // This task was already cancelled. In this case we should not
-                // insert the child into the list, since it would never get removed
-                // from the list.
-                (*child_token_state.synchronized.lock().unwrap()).is_cancelled = true;
-                unpacked_child_state.cancel_state = CancellationState::Cancelled;
-                // Since it's not in the list, the parent doesn't need to retain
-                // a reference to it.
-                unpacked_child_state.has_parent_ref = false;
-                child_token_state
-                    .state
-                    .store(unpacked_child_state.pack(), Ordering::SeqCst);
-            } else {
-                if let Some(mut first_child) = guard.first_child {
-                    child_token_state.from_parent.next_peer = Some(first_child);
-                    // Safety: We manipulate other child task inside the Mutex
-                    // and retain a parent reference on it. The child token can't
-                    // get invalidated while the Mutex is held.
-                    unsafe {
-                        first_child.as_mut().from_parent.prev_peer =
-                            Some((&mut *child_token_state).into())
-                    };
-                }
-                guard.first_child = Some((&mut *child_token_state).into());
-            }
-        };
-
-        let child_token_ptr = Box::into_raw(child_token_state);
-        // Safety: We just created the pointer from a `Box`
         CancellationToken {
-            inner: unsafe { NonNull::new_unchecked(child_token_ptr) },
+            inner: tree_node::child_node(&self.inner),
         }
     }
 
@@ -260,21 +153,33 @@ impl CancellationToken {
     /// derived from it.
     ///
     /// This will wake up all tasks which are waiting for cancellation.
+    ///
+    /// Be aware that cancellation is not an atomic operation. It is possible
+    /// for another thread running in parallel with a call to `cancel` to first
+    /// receive `true` from `is_cancelled` on one child node, and then receive
+    /// `false` from `is_cancelled` on another child node. However, once the
+    /// call to `cancel` returns, all child nodes have been fully cancelled.
     pub fn cancel(&self) {
-        self.state().cancel();
+        tree_node::cancel(&self.inner);
     }
 
-    /// Returns `true` if the `CancellationToken` had been cancelled
+    /// Returns `true` if the `CancellationToken` is cancelled.
     pub fn is_cancelled(&self) -> bool {
-        self.state().is_cancelled()
+        tree_node::is_cancelled(&self.inner)
     }
 
     /// Returns a `Future` that gets fulfilled when cancellation is requested.
+    ///
+    /// The future will complete immediately if the token is already cancelled
+    /// when this method is called.
+    ///
+    /// # Cancel safety
+    ///
+    /// This method is cancel safe.
     pub fn cancelled(&self) -> WaitForCancellationFuture<'_> {
         WaitForCancellationFuture {
-            cancellation_token: Some(self),
-            wait_node: ListNode::new(WaitQueueEntry::new()),
-            is_registered: false,
+            cancellation_token: self,
+            future: self.inner.notified(),
         }
     }
 
@@ -285,26 +190,6 @@ impl CancellationToken {
     pub fn drop_guard(self) -> DropGuard {
         DropGuard { inner: Some(self) }
     }
-
-    unsafe fn register(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        self.state().register(wait_node, cx)
-    }
-
-    fn check_for_cancellation(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        self.state().check_for_cancellation(wait_node, cx)
-    }
-
-    fn unregister(&self, wait_node: &mut ListNode<WaitQueueEntry>) {
-        self.state().unregister(wait_node)
-    }
 }
 
 // ===== impl WaitForCancellationFuture =====
@@ -319,560 +204,21 @@ impl<'a> Future for WaitForCancellationFuture<'a> {
     type Output = ();
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-        // Safety: We do not move anything out of `WaitForCancellationFuture`
-        let mut_self: &mut WaitForCancellationFuture<'_> = unsafe { Pin::get_unchecked_mut(self) };
-
-        let cancellation_token = mut_self
-            .cancellation_token
-            .expect("polled WaitForCancellationFuture after completion");
-
-        let poll_res = if !mut_self.is_registered {
-            // Safety: The `ListNode` is pinned through the Future,
-            // and we will unregister it in `WaitForCancellationFuture::drop`
-            // before the Future is dropped and the memory reference is invalidated.
-            unsafe { cancellation_token.register(&mut mut_self.wait_node, cx) }
-        } else {
-            cancellation_token.check_for_cancellation(&mut mut_self.wait_node, cx)
-        };
-
-        if let Poll::Ready(()) = poll_res {
-            // The cancellation_token was signalled
-            mut_self.cancellation_token = None;
-            // A signalled Token means the Waker won't be enqueued anymore
-            mut_self.is_registered = false;
-            mut_self.wait_node.task = None;
-        } else {
-            // This `Future` and its stored `Waker` stay registered at the
-            // `CancellationToken`
-            mut_self.is_registered = true;
-        }
-
-        poll_res
-    }
-}
-
-impl<'a> Drop for WaitForCancellationFuture<'a> {
-    fn drop(&mut self) {
-        // If this WaitForCancellationFuture has been polled and it was added to the
-        // wait queue at the cancellation_token, it must be removed before dropping.
-        // Otherwise the cancellation_token would access invalid memory.
-        if let Some(token) = self.cancellation_token {
-            if self.is_registered {
-                token.unregister(&mut self.wait_node);
-            }
-        }
-    }
-}
-
-/// Tracks how the future had interacted with the [`CancellationToken`]
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum PollState {
-    /// The task has never interacted with the [`CancellationToken`].
-    New,
-    /// The task was added to the wait queue at the [`CancellationToken`].
-    Waiting,
-    /// The task has been polled to completion.
-    Done,
-}
-
-/// Tracks the WaitForCancellationFuture waiting state.
-/// Access to this struct is synchronized through the mutex in the CancellationToken.
-struct WaitQueueEntry {
-    /// The task handle of the waiting task
-    task: Option<Waker>,
-    // Current polling state. This state is only updated inside the Mutex of
-    // the CancellationToken.
-    state: PollState,
-}
-
-impl WaitQueueEntry {
-    /// Creates a new WaitQueueEntry
-    fn new() -> WaitQueueEntry {
-        WaitQueueEntry {
-            task: None,
-            state: PollState::New,
-        }
-    }
-}
-
-struct SynchronizedState {
-    waiters: LinkedList<WaitQueueEntry>,
-    first_child: Option<NonNull<CancellationTokenState>>,
-    is_cancelled: bool,
-}
-
-impl SynchronizedState {
-    fn new() -> Self {
-        Self {
-            waiters: LinkedList::new(),
-            first_child: None,
-            is_cancelled: false,
-        }
-    }
-}
-
-/// Information embedded in child tokens which is synchronized through the Mutex
-/// in their parent.
-struct SynchronizedThroughParent {
-    next_peer: Option<NonNull<CancellationTokenState>>,
-    prev_peer: Option<NonNull<CancellationTokenState>>,
-}
-
-/// Possible states of a `CancellationToken`
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum CancellationState {
-    NotCancelled = 0,
-    Cancelling = 1,
-    Cancelled = 2,
-}
-
-impl CancellationState {
-    fn pack(self) -> usize {
-        self as usize
-    }
-
-    fn unpack(value: usize) -> Self {
-        match value {
-            0 => CancellationState::NotCancelled,
-            1 => CancellationState::Cancelling,
-            2 => CancellationState::Cancelled,
-            _ => unreachable!("Invalid value"),
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-struct StateSnapshot {
-    /// The amount of references to this particular CancellationToken.
-    /// `CancellationToken` structs hold these references to a `CancellationTokenState`.
-    /// Also the state is referenced by the state of each child.
-    refcount: usize,
-    /// Whether the state is still referenced by it's parent and can therefore
-    /// not be freed.
-    has_parent_ref: bool,
-    /// Whether the token is cancelled
-    cancel_state: CancellationState,
-}
-
-impl StateSnapshot {
-    /// Packs the snapshot into a `usize`
-    fn pack(self) -> usize {
-        self.refcount << 3 | if self.has_parent_ref { 4 } else { 0 } | self.cancel_state.pack()
-    }
-
-    /// Unpacks the snapshot from a `usize`
-    fn unpack(value: usize) -> Self {
-        let refcount = value >> 3;
-        let has_parent_ref = value & 4 != 0;
-        let cancel_state = CancellationState::unpack(value & 0x03);
-
-        StateSnapshot {
-            refcount,
-            has_parent_ref,
-            cancel_state,
-        }
-    }
-
-    /// Whether this `CancellationTokenState` is still referenced by any
-    /// `CancellationToken`.
-    fn has_refs(&self) -> bool {
-        self.refcount != 0 || self.has_parent_ref
-    }
-}
-
-/// The maximum permitted amount of references to a CancellationToken. This
-/// is derived from the intent to never use more than 32bit in the `Snapshot`.
-const MAX_REFS: u32 = (std::u32::MAX - 7) >> 3;
-
-/// Internal state of the `CancellationToken` pair above
-struct CancellationTokenState {
-    state: AtomicUsize,
-    parent: Option<NonNull<CancellationTokenState>>,
-    from_parent: SynchronizedThroughParent,
-    synchronized: Mutex<SynchronizedState>,
-}
-
-impl CancellationTokenState {
-    fn new(
-        parent: Option<NonNull<CancellationTokenState>>,
-        state: StateSnapshot,
-    ) -> CancellationTokenState {
-        CancellationTokenState {
-            parent,
-            from_parent: SynchronizedThroughParent {
-                prev_peer: None,
-                next_peer: None,
-            },
-            state: AtomicUsize::new(state.pack()),
-            synchronized: Mutex::new(SynchronizedState::new()),
-        }
-    }
-
-    /// Returns a snapshot of the current atomic state of the token
-    fn snapshot(&self) -> StateSnapshot {
-        StateSnapshot::unpack(self.state.load(Ordering::SeqCst))
-    }
-
-    fn atomic_update_state<F>(&self, mut current_state: StateSnapshot, func: F) -> StateSnapshot
-    where
-        F: Fn(StateSnapshot) -> StateSnapshot,
-    {
-        let mut current_packed_state = current_state.pack();
+        let mut this = self.project();
         loop {
-            let next_state = func(current_state);
-            match self.state.compare_exchange(
-                current_packed_state,
-                next_state.pack(),
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => {
-                    return next_state;
-                }
-                Err(actual) => {
-                    current_packed_state = actual;
-                    current_state = StateSnapshot::unpack(actual);
-                }
-            }
-        }
-    }
-
-    fn increment_refcount(&self, current_state: StateSnapshot) -> StateSnapshot {
-        self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            if state.refcount >= MAX_REFS as usize {
-                eprintln!("[ERROR] Maximum reference count for CancellationToken was exceeded");
-                std::process::abort();
-            }
-            state.refcount += 1;
-            state
-        })
-    }
-
-    fn decrement_refcount(&self, current_state: StateSnapshot) -> StateSnapshot {
-        let current_state = self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            state.refcount -= 1;
-            state
-        });
-
-        // Drop the State if it is not referenced anymore
-        if !current_state.has_refs() {
-            // Safety: `CancellationTokenState` is always stored in refcounted
-            // Boxes
-            let _ = unsafe { Box::from_raw(self as *const Self as *mut Self) };
-        }
-
-        current_state
-    }
-
-    fn remove_parent_ref(&self, current_state: StateSnapshot) -> StateSnapshot {
-        let current_state = self.atomic_update_state(current_state, |mut state: StateSnapshot| {
-            state.has_parent_ref = false;
-            state
-        });
-
-        // Drop the State if it is not referenced anymore
-        if !current_state.has_refs() {
-            // Safety: `CancellationTokenState` is always stored in refcounted
-            // Boxes
-            let _ = unsafe { Box::from_raw(self as *const Self as *mut Self) };
-        }
-
-        current_state
-    }
-
-    /// Unregisters a child from the parent token.
-    /// The child tokens state is not exactly known at this point in time.
-    /// If the parent token is cancelled, the child token gets removed from the
-    /// parents list, and might therefore already have been freed. If the parent
-    /// token is not cancelled, the child token is still valid.
-    fn unregister_child(
-        &mut self,
-        mut child_state: NonNull<CancellationTokenState>,
-        current_child_state: StateSnapshot,
-    ) {
-        let removed_child = {
-            // Remove the child toke from the parents linked list
-            let mut guard = self.synchronized.lock().unwrap();
-            if !guard.is_cancelled {
-                // Safety: Since the token was not cancelled, the child must
-                // still be in the list and valid.
-                let mut child_state = unsafe { child_state.as_mut() };
-                debug_assert!(child_state.snapshot().has_parent_ref);
-
-                if guard.first_child == Some(child_state.into()) {
-                    guard.first_child = child_state.from_parent.next_peer;
-                }
-                // Safety: If peers wouldn't be valid anymore, they would try
-                // to remove themselves from the list. This would require locking
-                // the Mutex that we currently own.
-                unsafe {
-                    if let Some(mut prev_peer) = child_state.from_parent.prev_peer {
-                        prev_peer.as_mut().from_parent.next_peer =
-                            child_state.from_parent.next_peer;
-                    }
-                    if let Some(mut next_peer) = child_state.from_parent.next_peer {
-                        next_peer.as_mut().from_parent.prev_peer =
-                            child_state.from_parent.prev_peer;
-                    }
-                }
-                child_state.from_parent.prev_peer = None;
-                child_state.from_parent.next_peer = None;
-
-                // The child is no longer referenced by the parent, since we were able
-                // to remove its reference from the parents list.
-                true
-            } else {
-                // Do not touch the linked list anymore. If the parent is cancelled
-                // it will move all childs outside of the Mutex and manipulate
-                // the pointers there. Manipulating the pointers here too could
-                // lead to races. Therefore leave them just as as and let the
-                // parent deal with it. The parent will make sure to retain a
-                // reference to this state as long as it manipulates the list
-                // pointers. Therefore the pointers are not dangling.
-                false
-            }
-        };
-
-        if removed_child {
-            // If the token removed itself from the parents list, it can reset
-            // the parent ref status. If it is isn't able to do so, because the
-            // parent removed it from the list, there is no need to do this.
-            // The parent ref acts as as another reference count. Therefore
-            // removing this reference can free the object.
-            // Safety: The token was in the list. This means the parent wasn't
-            // cancelled before, and the token must still be alive.
-            unsafe { child_state.as_mut().remove_parent_ref(current_child_state) };
-        }
-
-        // Decrement the refcount on the parent and free it if necessary
-        self.decrement_refcount(self.snapshot());
-    }
-
-    fn cancel(&self) {
-        // Move the state of the CancellationToken from `NotCancelled` to `Cancelling`
-        let mut current_state = self.snapshot();
-
-        let state_after_cancellation = loop {
-            if current_state.cancel_state != CancellationState::NotCancelled {
-                // Another task already initiated the cancellation
-                return;
+            if this.cancellation_token.is_cancelled() {
+                return Poll::Ready(());
             }
 
-            let mut next_state = current_state;
-            next_state.cancel_state = CancellationState::Cancelling;
-            match self.state.compare_exchange(
-                current_state.pack(),
-                next_state.pack(),
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => break next_state,
-                Err(actual) => current_state = StateSnapshot::unpack(actual),
+            // No wakeups can be lost here because there is always a call to
+            // `is_cancelled` between the creation of the future and the call to
+            // `poll`, and the code that sets the cancelled flag does so before
+            // waking the `Notified`.
+            if this.future.as_mut().poll(cx).is_pending() {
+                return Poll::Pending;
             }
-        };
 
-        // This task cancelled the token
-
-        // Take the task list out of the Token
-        // We do not want to cancel child token inside this lock. If one of the
-        // child tasks would have additional child tokens, we would recursively
-        // take locks.
-
-        // Doing this action has an impact if the child token is dropped concurrently:
-        // It will try to deregister itself from the parent task, but can not find
-        // itself in the task list anymore. Therefore it needs to assume the parent
-        // has extracted the list and will process it. It may not modify the list.
-        // This is OK from a memory safety perspective, since the parent still
-        // retains a reference to the child task until it finished iterating over
-        // it.
-
-        let mut first_child = {
-            let mut guard = self.synchronized.lock().unwrap();
-            // Save the cancellation also inside the Mutex
-            // This allows child tokens which want to detach themselves to detect
-            // that this is no longer required since the parent cleared the list.
-            guard.is_cancelled = true;
-
-            // Wakeup all waiters
-            // This happens inside the lock to make cancellation reliable
-            // If we would access waiters outside of the lock, the pointers
-            // may no longer be valid.
-            // Typically this shouldn't be an issue, since waking a task should
-            // only move it from the blocked into the ready state and not have
-            // further side effects.
-
-            // Use a reverse iterator, so that the oldest waiter gets
-            // scheduled first
-            guard.waiters.reverse_drain(|waiter| {
-                // We are not allowed to move the `Waker` out of the list node.
-                // The `Future` relies on the fact that the old `Waker` stays there
-                // as long as the `Future` has not completed in order to perform
-                // the `will_wake()` check.
-                // Therefore `wake_by_ref` is used instead of `wake()`
-                if let Some(handle) = &mut waiter.task {
-                    handle.wake_by_ref();
-                }
-                // Mark the waiter to have been removed from the list.
-                waiter.state = PollState::Done;
-            });
-
-            guard.first_child.take()
-        };
-
-        while let Some(mut child) = first_child {
-            // Safety: We know this is a valid pointer since it is in our child pointer
-            // list. It can't have been freed in between, since we retain a a reference
-            // to each child.
-            let mut_child = unsafe { child.as_mut() };
-
-            // Get the next child and clean up list pointers
-            first_child = mut_child.from_parent.next_peer;
-            mut_child.from_parent.prev_peer = None;
-            mut_child.from_parent.next_peer = None;
-
-            // Cancel the child task
-            mut_child.cancel();
-
-            // Drop the parent reference. This `CancellationToken` is not interested
-            // in interacting with the child anymore.
-            // This is ONLY allowed once we promised not to touch the state anymore
-            // after this interaction.
-            mut_child.remove_parent_ref(mut_child.snapshot());
+            this.future.set(this.cancellation_token.inner.notified());
         }
-
-        // The cancellation has completed
-        // At this point in time tasks which registered a wait node can be sure
-        // that this wait node already had been dequeued from the list without
-        // needing to inspect the list.
-        self.atomic_update_state(state_after_cancellation, |mut state| {
-            state.cancel_state = CancellationState::Cancelled;
-            state
-        });
-    }
-
-    /// Returns `true` if the `CancellationToken` had been cancelled
-    fn is_cancelled(&self) -> bool {
-        let current_state = self.snapshot();
-        current_state.cancel_state != CancellationState::NotCancelled
-    }
-
-    /// Registers a waiting task at the `CancellationToken`.
-    /// Safety: This method is only safe as long as the waiting waiting task
-    /// will properly unregister the wait node before it gets moved.
-    unsafe fn register(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        debug_assert_eq!(PollState::New, wait_node.state);
-        let current_state = self.snapshot();
-
-        // Perform an optimistic cancellation check before. This is not strictly
-        // necessary since we also check for cancellation in the Mutex, but
-        // reduces the necessary work to be performed for tasks which already
-        // had been cancelled.
-        if current_state.cancel_state != CancellationState::NotCancelled {
-            return Poll::Ready(());
-        }
-
-        // So far the token is not cancelled. However it could be cancelled before
-        // we get the chance to store the `Waker`. Therefore we need to check
-        // for cancellation again inside the mutex.
-        let mut guard = self.synchronized.lock().unwrap();
-        if guard.is_cancelled {
-            // Cancellation was signalled
-            wait_node.state = PollState::Done;
-            Poll::Ready(())
-        } else {
-            // Added the task to the wait queue
-            wait_node.task = Some(cx.waker().clone());
-            wait_node.state = PollState::Waiting;
-            guard.waiters.add_front(wait_node);
-            Poll::Pending
-        }
-    }
-
-    fn check_for_cancellation(
-        &self,
-        wait_node: &mut ListNode<WaitQueueEntry>,
-        cx: &mut Context<'_>,
-    ) -> Poll<()> {
-        debug_assert!(
-            wait_node.task.is_some(),
-            "Method can only be called after task had been registered"
-        );
-
-        let current_state = self.snapshot();
-
-        if current_state.cancel_state != CancellationState::NotCancelled {
-            // If the cancellation had been fully completed we know that our `Waker`
-            // is no longer registered at the `CancellationToken`.
-            // Otherwise the cancel call may or may not yet have iterated
-            // through the waiters list and removed the wait nodes.
-            // If it hasn't yet, we need to remove it. Otherwise an attempt to
-            // reuse the `wait_node´ might get freed due to the `WaitForCancellationFuture`
-            // getting dropped before the cancellation had interacted with it.
-            if current_state.cancel_state != CancellationState::Cancelled {
-                self.unregister(wait_node);
-            }
-            Poll::Ready(())
-        } else {
-            // Check if we need to swap the `Waker`. This will make the check more
-            // expensive, since the `Waker` is synchronized through the Mutex.
-            // If we don't need to perform a `Waker` update, an atomic check for
-            // cancellation is sufficient.
-            let need_waker_update = wait_node
-                .task
-                .as_ref()
-                .map(|waker| waker.will_wake(cx.waker()))
-                .unwrap_or(true);
-
-            if need_waker_update {
-                let guard = self.synchronized.lock().unwrap();
-                if guard.is_cancelled {
-                    // Cancellation was signalled. Since this cancellation signal
-                    // is set inside the Mutex, the old waiter must already have
-                    // been removed from the waiting list
-                    debug_assert_eq!(PollState::Done, wait_node.state);
-                    wait_node.task = None;
-                    Poll::Ready(())
-                } else {
-                    // The WaitForCancellationFuture is already in the queue.
-                    // The CancellationToken can't have been cancelled,
-                    // since this would change the is_cancelled flag inside the mutex.
-                    // Therefore we just have to update the Waker. A follow-up
-                    // cancellation will always use the new waker.
-                    wait_node.task = Some(cx.waker().clone());
-                    Poll::Pending
-                }
-            } else {
-                // Do nothing. If the token gets cancelled, this task will get
-                // woken again and can fetch the cancellation.
-                Poll::Pending
-            }
-        }
-    }
-
-    fn unregister(&self, wait_node: &mut ListNode<WaitQueueEntry>) {
-        debug_assert!(
-            wait_node.task.is_some(),
-            "waiter can not be active without task"
-        );
-
-        let mut guard = self.synchronized.lock().unwrap();
-        // WaitForCancellationFuture only needs to get removed if it has been added to
-        // the wait queue of the CancellationToken.
-        // This has happened in the PollState::Waiting case.
-        if let PollState::Waiting = wait_node.state {
-            // Safety: Due to the state, we know that the node must be part
-            // of the waiter list
-            if !unsafe { guard.waiters.remove(wait_node) } {
-                // Panic if the address isn't found. This can only happen if the contract was
-                // violated, e.g. the WaitQueueEntry got moved after the initial poll.
-                panic!("Future could not be removed from wait queue");
-            }
-            wait_node.state = PollState::Done;
-        }
-        wait_node.task = None;
     }
 }

tokio-util/src/sync/cancellation_token/tree_node.rs

@@ -0,0 +1,373 @@
+//! This mod provides the logic for the inner tree structure of the CancellationToken.
+//!
+//! CancellationTokens are only light handles with references to TreeNode.
+//! All the logic is actually implemented in the TreeNode.
+//!
+//! A TreeNode is part of the cancellation tree and may have one parent and an arbitrary number of
+//! children.
+//!
+//! A TreeNode can receive the request to perform a cancellation through a CancellationToken.
+//! This cancellation request will cancel the node and all of its descendants.
+//!
+//! As soon as a node cannot get cancelled any more (because it was already cancelled or it has no
+//! more CancellationTokens pointing to it any more), it gets removed from the tree, to keep the
+//! tree as small as possible.
+//!
+//! # Invariants
+//!
+//! Those invariants shall be true at any time.
+//!
+//! 1. A node that has no parents and no handles can no longer be cancelled.
+//!     This is important during both cancellation and refcounting.
+//!
+//! 2. If node B *is* or *was* a child of node A, then node B was created *after* node A.
+//!     This is important for deadlock safety, as it is used for lock order.
+//!     Node B can only become the child of node A in two ways:
+//!         - being created with `child_node()`, in which case it is trivially true that
+//!           node A already existed when node B was created
+//!         - being moved A->C->B to A->B because node C was removed in `decrease_handle_refcount()`
+//!           or `cancel()`. In this case the invariant still holds, as B was younger than C, and C
+//!           was younger than A, therefore B is also younger than A.
+//!
+//! 3. If two nodes are both unlocked and node A is the parent of node B, then node B is a child of
+//!    node A. It is important to always restore that invariant before dropping the lock of a node.
+//!
+//! # Deadlock safety
+//!
+//! We always lock in the order of creation time. We can prove this through invariant #2.
+//! Specifically, through invariant #2, we know that we always have to lock a parent
+//! before its child.
+//!
+use crate::loom::sync::{Arc, Mutex, MutexGuard};
+
+/// A node of the cancellation tree structure
+///
+/// The actual data it holds is wrapped inside a mutex for synchronization.
+pub(crate) struct TreeNode {
+    inner: Mutex<Inner>,
+    waker: tokio::sync::Notify,
+}
+impl TreeNode {
+    pub(crate) fn new() -> Self {
+        Self {
+            inner: Mutex::new(Inner {
+                parent: None,
+                parent_idx: 0,
+                children: vec![],
+                is_cancelled: false,
+                num_handles: 1,
+            }),
+            waker: tokio::sync::Notify::new(),
+        }
+    }
+
+    pub(crate) fn notified(&self) -> tokio::sync::futures::Notified<'_> {
+        self.waker.notified()
+    }
+}
+
+/// The data contained inside a TreeNode.
+///
+/// This struct exists so that the data of the node can be wrapped
+/// in a Mutex.
+struct Inner {
+    parent: Option<Arc<TreeNode>>,
+    parent_idx: usize,
+    children: Vec<Arc<TreeNode>>,
+    is_cancelled: bool,
+    num_handles: usize,
+}
+
+/// Returns whether or not the node is cancelled
+pub(crate) fn is_cancelled(node: &Arc<TreeNode>) -> bool {
+    node.inner.lock().unwrap().is_cancelled
+}
+
+/// Creates a child node
+pub(crate) fn child_node(parent: &Arc<TreeNode>) -> Arc<TreeNode> {
+    let mut locked_parent = parent.inner.lock().unwrap();
+
+    // Do not register as child if we are already cancelled.
+    // Cancelled trees can never be uncancelled and therefore
+    // need no connection to parents or children any more.
+    if locked_parent.is_cancelled {
+        return Arc::new(TreeNode {
+            inner: Mutex::new(Inner {
+                parent: None,
+                parent_idx: 0,
+                children: vec![],
+                is_cancelled: true,
+                num_handles: 1,
+            }),
+            waker: tokio::sync::Notify::new(),
+        });
+    }
+
+    let child = Arc::new(TreeNode {
+        inner: Mutex::new(Inner {
+            parent: Some(parent.clone()),
+            parent_idx: locked_parent.children.len(),
+            children: vec![],
+            is_cancelled: false,
+            num_handles: 1,
+        }),
+        waker: tokio::sync::Notify::new(),
+    });
+
+    locked_parent.children.push(child.clone());
+
+    child
+}
+
+/// Disconnects the given parent from all of its children.
+///
+/// Takes a reference to [Inner] to make sure the parent is already locked.
+fn disconnect_children(node: &mut Inner) {
+    for child in std::mem::take(&mut node.children) {
+        let mut locked_child = child.inner.lock().unwrap();
+        locked_child.parent_idx = 0;
+        locked_child.parent = None;
+    }
+}
+
+/// Figures out the parent of the node and locks the node and its parent atomically.
+///
+/// The basic principle of preventing deadlocks in the tree is
+/// that we always lock the parent first, and then the child.
+/// For more info look at *deadlock safety* and *invariant #2*.
+///
+/// Sadly, it's impossible to figure out the parent of a node without
+/// locking it. To then achieve locking order consistency, the node
+/// has to be unlocked before the parent gets locked.
+/// This leaves a small window where we already assume that we know the parent,
+/// but neither the parent nor the node is locked. Therefore, the parent could change.
+///
+/// To prevent that this problem leaks into the rest of the code, it is abstracted
+/// in this function.
+///
+/// The locked child and optionally its locked parent, if a parent exists, get passed
+/// to the `func` argument via (node, None) or (node, Some(parent)).
+fn with_locked_node_and_parent<F, Ret>(node: &Arc<TreeNode>, func: F) -> Ret
+where
+    F: FnOnce(MutexGuard<'_, Inner>, Option<MutexGuard<'_, Inner>>) -> Ret,
+{
+    let mut potential_parent = {
+        let locked_node = node.inner.lock().unwrap();
+        match locked_node.parent.clone() {
+            Some(parent) => parent,
+            // If we locked the node and its parent is `None`, we are in a valid state
+            // and can return.
+            None => return func(locked_node, None),
+        }
+    };
+
+    loop {
+        // Deadlock safety:
+        //
+        // Due to invariant #2, we know that we have to lock the parent first, and then the child.
+        // This is true even if the potential_parent is no longer the current parent or even its
+        // sibling, as the invariant still holds.
+        let locked_parent = potential_parent.inner.lock().unwrap();
+        let locked_node = node.inner.lock().unwrap();
+
+        let actual_parent = match locked_node.parent.clone() {
+            Some(parent) => parent,
+            // If we locked the node and its parent is `None`, we are in a valid state
+            // and can return.
+            None => {
+                // Was the wrong parent, so unlock it before calling `func`
+                drop(locked_parent);
+                return func(locked_node, None);
+            }
+        };
+
+        // Loop until we managed to lock both the node and its parent
+        if Arc::ptr_eq(&actual_parent, &potential_parent) {
+            return func(locked_node, Some(locked_parent));
+        }
+
+        // Drop locked_parent before reassigning to potential_parent,
+        // as potential_parent is borrowed in it
+        drop(locked_node);
+        drop(locked_parent);
+
+        potential_parent = actual_parent;
+    }
+}
+
+/// Moves all children from `node` to `parent`.
+///
+/// `parent` MUST have been a parent of the node when they both got locked,
+/// otherwise there is a potential for a deadlock as invariant #2 would be violated.
+///
+/// To aquire the locks for node and parent, use [with_locked_node_and_parent].
+fn move_children_to_parent(node: &mut Inner, parent: &mut Inner) {
+    // Pre-allocate in the parent, for performance
+    parent.children.reserve(node.children.len());
+
+    for child in std::mem::take(&mut node.children) {
+        {
+            let mut child_locked = child.inner.lock().unwrap();
+            child_locked.parent = node.parent.clone();
+            child_locked.parent_idx = parent.children.len();
+        }
+        parent.children.push(child);
+    }
+}
+
+/// Removes a child from the parent.
+///
+/// `parent` MUST be the parent of `node`.
+/// To aquire the locks for node and parent, use [with_locked_node_and_parent].
+fn remove_child(parent: &mut Inner, mut node: MutexGuard<'_, Inner>) {
+    // Query the position from where to remove a node
+    let pos = node.parent_idx;
+    node.parent = None;
+    node.parent_idx = 0;
+
+    // Unlock node, so that only one child at a time is locked.
+    // Otherwise we would violate the lock order (see 'deadlock safety') as we
+    // don't know the creation order of the child nodes
+    drop(node);
+
+    // If `node` is the last element in the list, we don't need any swapping
+    if parent.children.len() == pos + 1 {
+        parent.children.pop().unwrap();
+    } else {
+        // If `node` is not the last element in the list, we need to
+        // replace it with the last element
+        let replacement_child = parent.children.pop().unwrap();
+        replacement_child.inner.lock().unwrap().parent_idx = pos;
+        parent.children[pos] = replacement_child;
+    }
+
+    let len = parent.children.len();
+    if 4 * len <= parent.children.capacity() {
+        // equal to:
+        //    parent.children.shrink_to(2 * len);
+        // but shrink_to was not yet stabilized in our minimal compatible version
+        let old_children = std::mem::replace(&mut parent.children, Vec::with_capacity(2 * len));
+        parent.children.extend(old_children);
+    }
+}
+
+/// Increases the reference count of handles.
+pub(crate) fn increase_handle_refcount(node: &Arc<TreeNode>) {
+    let mut locked_node = node.inner.lock().unwrap();
+
+    // Once no handles are left over, the node gets detached from the tree.
+    // There should never be a new handle once all handles are dropped.
+    assert!(locked_node.num_handles > 0);
+
+    locked_node.num_handles += 1;
+}
+
+/// Decreases the reference count of handles.
+///
+/// Once no handle is left, we can remove the node from the
+/// tree and connect its parent directly to its children.
+pub(crate) fn decrease_handle_refcount(node: &Arc<TreeNode>) {
+    let num_handles = {
+        let mut locked_node = node.inner.lock().unwrap();
+        locked_node.num_handles -= 1;
+        locked_node.num_handles
+    };
+
+    if num_handles == 0 {
+        with_locked_node_and_parent(node, |mut node, parent| {
+            // Remove the node from the tree
+            match parent {
+                Some(mut parent) => {
+                    // As we want to remove ourselves from the tree,
+                    // we have to move the children to the parent, so that
+                    // they still receive the cancellation event without us.
+                    // Moving them does not violate invariant #1.
+                    move_children_to_parent(&mut node, &mut parent);
+
+                    // Remove the node from the parent
+                    remove_child(&mut parent, node);
+                }
+                None => {
+                    // Due to invariant #1, we can assume that our
+                    // children can no longer be cancelled through us.
+                    // (as we now have neither a parent nor handles)
+                    // Therefore we can disconnect them.
+                    disconnect_children(&mut node);
+                }
+            }
+        });
+    }
+}
+
+/// Cancels a node and its children.
+pub(crate) fn cancel(node: &Arc<TreeNode>) {
+    let mut locked_node = node.inner.lock().unwrap();
+
+    if locked_node.is_cancelled {
+        return;
+    }
+
+    // One by one, adopt grandchildren and then cancel and detach the child
+    while let Some(child) = locked_node.children.pop() {
+        // This can't deadlock because the mutex we are already
+        // holding is the parent of child.
+        let mut locked_child = child.inner.lock().unwrap();
+
+        // Detach the child from node
+        // No need to modify node.children, as the child already got removed with `.pop`
+        locked_child.parent = None;
+        locked_child.parent_idx = 0;
+
+        // If child is already cancelled, detaching is enough
+        if locked_child.is_cancelled {
+            continue;
+        }
+
+        // Cancel or adopt grandchildren
+        while let Some(grandchild) = locked_child.children.pop() {
+            // This can't deadlock because the two mutexes we are already
+            // holding is the parent and grandparent of grandchild.
+            let mut locked_grandchild = grandchild.inner.lock().unwrap();
+
+            // Detach the grandchild
+            locked_grandchild.parent = None;
+            locked_grandchild.parent_idx = 0;
+
+            // If grandchild is already cancelled, detaching is enough
+            if locked_grandchild.is_cancelled {
+                continue;
+            }
+
+            // For performance reasons, only adopt grandchildren that have children.
+            // Otherwise, just cancel them right away, no need for another iteration.
+            if locked_grandchild.children.is_empty() {
+                // Cancel the grandchild
+                locked_grandchild.is_cancelled = true;
+                locked_grandchild.children = Vec::new();
+                drop(locked_grandchild);
+                grandchild.waker.notify_waiters();
+            } else {
+                // Otherwise, adopt grandchild
+                locked_grandchild.parent = Some(node.clone());
+                locked_grandchild.parent_idx = locked_node.children.len();
+                drop(locked_grandchild);
+                locked_node.children.push(grandchild);
+            }
+        }
+
+        // Cancel the child
+        locked_child.is_cancelled = true;
+        locked_child.children = Vec::new();
+        drop(locked_child);
+        child.waker.notify_waiters();
+
+        // Now the child is cancelled and detached and all its children are adopted.
+        // Just continue until all (including adopted) children are cancelled and detached.
+    }
+
+    // Cancel the node itself.
+    locked_node.is_cancelled = true;
+    locked_node.children = Vec::new();
+    drop(locked_node);
+    node.waker.notify_waiters();
+}

tokio-util/src/sync/intrusive_double_linked_list.rs

@@ -1,788 +0,0 @@
-//! An intrusive double linked list of data
-
-#![allow(dead_code, unreachable_pub)]
-
-use core::{
-    marker::PhantomPinned,
-    ops::{Deref, DerefMut},
-    ptr::NonNull,
-};
-
-/// A node which carries data of type `T` and is stored in an intrusive list
-#[derive(Debug)]
-pub struct ListNode<T> {
-    /// The previous node in the list. `None` if there is no previous node.
-    prev: Option<NonNull<ListNode<T>>>,
-    /// The next node in the list. `None` if there is no previous node.
-    next: Option<NonNull<ListNode<T>>>,
-    /// The data which is associated to this list item
-    data: T,
-    /// Prevents `ListNode`s from being `Unpin`. They may never be moved, since
-    /// the list semantics require addresses to be stable.
-    _pin: PhantomPinned,
-}
-
-impl<T> ListNode<T> {
-    /// Creates a new node with the associated data
-    pub fn new(data: T) -> ListNode<T> {
-        Self {
-            prev: None,
-            next: None,
-            data,
-            _pin: PhantomPinned,
-        }
-    }
-}
-
-impl<T> Deref for ListNode<T> {
-    type Target = T;
-
-    fn deref(&self) -> &T {
-        &self.data
-    }
-}
-
-impl<T> DerefMut for ListNode<T> {
-    fn deref_mut(&mut self) -> &mut T {
-        &mut self.data
-    }
-}
-
-/// An intrusive linked list of nodes, where each node carries associated data
-/// of type `T`.
-#[derive(Debug)]
-pub struct LinkedList<T> {
-    head: Option<NonNull<ListNode<T>>>,
-    tail: Option<NonNull<ListNode<T>>>,
-}
-
-impl<T> LinkedList<T> {
-    /// Creates an empty linked list
-    pub fn new() -> Self {
-        LinkedList::<T> {
-            head: None,
-            tail: None,
-        }
-    }
-
-    /// Adds a node at the front of the linked list.
-    /// Safety: This function is only safe as long as `node` is guaranteed to
-    /// get removed from the list before it gets moved or dropped.
-    /// In addition to this `node` may not be added to another other list before
-    /// it is removed from the current one.
-    pub unsafe fn add_front(&mut self, node: &mut ListNode<T>) {
-        node.next = self.head;
-        node.prev = None;
-        if let Some(mut head) = self.head {
-            head.as_mut().prev = Some(node.into())
-        };
-        self.head = Some(node.into());
-        if self.tail.is_none() {
-            self.tail = Some(node.into());
-        }
-    }
-
-    /// Inserts a node into the list in a way that the list keeps being sorted.
-    /// Safety: This function is only safe as long as `node` is guaranteed to
-    /// get removed from the list before it gets moved or dropped.
-    /// In addition to this `node` may not be added to another other list before
-    /// it is removed from the current one.
-    pub unsafe fn add_sorted(&mut self, node: &mut ListNode<T>)
-    where
-        T: PartialOrd,
-    {
-        if self.head.is_none() {
-            // First node in the list
-            self.head = Some(node.into());
-            self.tail = Some(node.into());
-            return;
-        }
-
-        let mut prev: Option<NonNull<ListNode<T>>> = None;
-        let mut current = self.head;
-
-        while let Some(mut current_node) = current {
-            if node.data < current_node.as_ref().data {
-                // Need to insert before the current node
-                current_node.as_mut().prev = Some(node.into());
-                match prev {
-                    Some(mut prev) => {
-                        prev.as_mut().next = Some(node.into());
-                    }
-                    None => {
-                        // We are inserting at the beginning of the list
-                        self.head = Some(node.into());
-                    }
-                }
-                node.next = current;
-                node.prev = prev;
-                return;
-            }
-            prev = current;
-            current = current_node.as_ref().next;
-        }
-
-        // We looped through the whole list and the nodes data is bigger or equal
-        // than everything we found up to now.
-        // Insert at the end. Since we checked before that the list isn't empty,
-        // tail always has a value.
-        node.prev = self.tail;
-        node.next = None;
-        self.tail.as_mut().unwrap().as_mut().next = Some(node.into());
-        self.tail = Some(node.into());
-    }
-
-    /// Returns the first node in the linked list without removing it from the list
-    /// The function is only safe as long as valid pointers are stored inside
-    /// the linked list.
-    /// The returned pointer is only guaranteed to be valid as long as the list
-    /// is not mutated
-    pub fn peek_first(&self) -> Option<&mut ListNode<T>> {
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list.
-        // The returned node has a pointer which constrains it to the lifetime
-        // of the list. This is ok, since the Node is supposed to outlive
-        // its insertion in the list.
-        unsafe {
-            self.head
-                .map(|mut node| &mut *(node.as_mut() as *mut ListNode<T>))
-        }
-    }
-
-    /// Returns the last node in the linked list without removing it from the list
-    /// The function is only safe as long as valid pointers are stored inside
-    /// the linked list.
-    /// The returned pointer is only guaranteed to be valid as long as the list
-    /// is not mutated
-    pub fn peek_last(&self) -> Option<&mut ListNode<T>> {
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list.
-        // The returned node has a pointer which constrains it to the lifetime
-        // of the list. This is ok, since the Node is supposed to outlive
-        // its insertion in the list.
-        unsafe {
-            self.tail
-                .map(|mut node| &mut *(node.as_mut() as *mut ListNode<T>))
-        }
-    }
-
-    /// Removes the first node from the linked list
-    pub fn remove_first(&mut self) -> Option<&mut ListNode<T>> {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list
-        unsafe {
-            let mut head = self.head?;
-            self.head = head.as_mut().next;
-
-            let first_ref = head.as_mut();
-            match first_ref.next {
-                None => {
-                    // This was the only node in the list
-                    debug_assert_eq!(Some(first_ref.into()), self.tail);
-                    self.tail = None;
-                }
-                Some(mut next) => {
-                    next.as_mut().prev = None;
-                }
-            }
-
-            first_ref.prev = None;
-            first_ref.next = None;
-            Some(&mut *(first_ref as *mut ListNode<T>))
-        }
-    }
-
-    /// Removes the last node from the linked list and returns it
-    pub fn remove_last(&mut self) -> Option<&mut ListNode<T>> {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        // Safety: When the node was inserted it was promised that it is alive
-        // until it gets removed from the list
-        unsafe {
-            let mut tail = self.tail?;
-            self.tail = tail.as_mut().prev;
-
-            let last_ref = tail.as_mut();
-            match last_ref.prev {
-                None => {
-                    // This was the last node in the list
-                    debug_assert_eq!(Some(last_ref.into()), self.head);
-                    self.head = None;
-                }
-                Some(mut prev) => {
-                    prev.as_mut().next = None;
-                }
-            }
-
-            last_ref.prev = None;
-            last_ref.next = None;
-            Some(&mut *(last_ref as *mut ListNode<T>))
-        }
-    }
-
-    /// Returns whether the linked list does not contain any node
-    pub fn is_empty(&self) -> bool {
-        if self.head.is_some() {
-            return false;
-        }
-
-        debug_assert!(self.tail.is_none());
-        true
-    }
-
-    /// Removes the given `node` from the linked list.
-    /// Returns whether the `node` was removed.
-    /// It is also only safe if it is known that the `node` is either part of this
-    /// list, or of no list at all. If `node` is part of another list, the
-    /// behavior is undefined.
-    pub unsafe fn remove(&mut self, node: &mut ListNode<T>) -> bool {
-        #![allow(clippy::debug_assert_with_mut_call)]
-
-        match node.prev {
-            None => {
-                // This might be the first node in the list. If it is not, the
-                // node is not in the list at all. Since our precondition is that
-                // the node must either be in this list or in no list, we check that
-                // the node is really in no list.
-                if self.head != Some(node.into()) {
-                    debug_assert!(node.next.is_none());
-                    return false;
-                }
-                self.head = node.next;
-            }
-            Some(mut prev) => {
-                debug_assert_eq!(prev.as_ref().next, Some(node.into()));
-                prev.as_mut().next = node.next;
-            }
-        }
-
-        match node.next {
-            None => {
-                // This must be the last node in our list. Otherwise the list
-                // is inconsistent.
-                debug_assert_eq!(self.tail, Some(node.into()));
-                self.tail = node.prev;
-            }
-            Some(mut next) => {
-                debug_assert_eq!(next.as_mut().prev, Some(node.into()));
-                next.as_mut().prev = node.prev;
-            }
-        }
-
-        node.next = None;
-        node.prev = None;
-
-        true
-    }
-
-    /// Drains the list iby calling a callback on each list node
-    ///
-    /// The method does not return an iterator since stopping or deferring
-    /// draining the list is not permitted. If the method would push nodes to
-    /// an iterator we could not guarantee that the nodes do not get utilized
-    /// after having been removed from the list anymore.
-    pub fn drain<F>(&mut self, mut func: F)
-    where
-        F: FnMut(&mut ListNode<T>),
-    {
-        let mut current = self.head;
-        self.head = None;
-        self.tail = None;
-
-        while let Some(mut node) = current {
-            // Safety: The nodes have not been removed from the list yet and must
-            // therefore contain valid data. The nodes can also not be added to
-            // the list again during iteration, since the list is mutably borrowed.
-            unsafe {
-                let node_ref = node.as_mut();
-                current = node_ref.next;
-
-                node_ref.next = None;
-                node_ref.prev = None;
-
-                // Note: We do not reset the pointers from the next element in the
-                // list to the current one since we will iterate over the whole
-                // list anyway, and therefore clean up all pointers.
-
-                func(node_ref);
-            }
-        }
-    }
-
-    /// Drains the list in reverse order by calling a callback on each list node
-    ///
-    /// The method does not return an iterator since stopping or deferring
-    /// draining the list is not permitted. If the method would push nodes to
-    /// an iterator we could not guarantee that the nodes do not get utilized
-    /// after having been removed from the list anymore.
-    pub fn reverse_drain<F>(&mut self, mut func: F)
-    where
-        F: FnMut(&mut ListNode<T>),
-    {
-        let mut current = self.tail;
-        self.head = None;
-        self.tail = None;
-
-        while let Some(mut node) = current {
-            // Safety: The nodes have not been removed from the list yet and must
-            // therefore contain valid data. The nodes can also not be added to
-            // the list again during iteration, since the list is mutably borrowed.
-            unsafe {
-                let node_ref = node.as_mut();
-                current = node_ref.prev;
-
-                node_ref.next = None;
-                node_ref.prev = None;
-
-                // Note: We do not reset the pointers from the next element in the
-                // list to the current one since we will iterate over the whole
-                // list anyway, and therefore clean up all pointers.
-
-                func(node_ref);
-            }
-        }
-    }
-}
-
-#[cfg(all(test, feature = "std"))] // Tests make use of Vec at the moment
-mod tests {
-    use super::*;
-
-    fn collect_list<T: Copy>(mut list: LinkedList<T>) -> Vec<T> {
-        let mut result = Vec::new();
-        list.drain(|node| {
-            result.push(**node);
-        });
-        result
-    }
-
-    fn collect_reverse_list<T: Copy>(mut list: LinkedList<T>) -> Vec<T> {
-        let mut result = Vec::new();
-        list.reverse_drain(|node| {
-            result.push(**node);
-        });
-        result
-    }
-
-    unsafe fn add_nodes(list: &mut LinkedList<i32>, nodes: &mut [&mut ListNode<i32>]) {
-        for node in nodes.iter_mut() {
-            list.add_front(node);
-        }
-    }
-
-    unsafe fn assert_clean<T>(node: &mut ListNode<T>) {
-        assert!(node.next.is_none());
-        assert!(node.prev.is_none());
-    }
-
-    #[test]
-    fn insert_and_iterate() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut setup = |list: &mut LinkedList<i32>| {
-                assert_eq!(true, list.is_empty());
-                list.add_front(&mut c);
-                assert_eq!(31, **list.peek_first().unwrap());
-                assert_eq!(false, list.is_empty());
-                list.add_front(&mut b);
-                assert_eq!(7, **list.peek_first().unwrap());
-                list.add_front(&mut a);
-                assert_eq!(5, **list.peek_first().unwrap());
-            };
-
-            let mut list = LinkedList::new();
-            setup(&mut list);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            setup(&mut list);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([31, 7, 5].to_vec(), items);
-        }
-    }
-
-    #[test]
-    fn add_sorted() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-            let mut d = ListNode::new(99);
-
-            let mut list = LinkedList::new();
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut b]);
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut b]);
-            list.add_sorted(&mut a);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut a]);
-            list.add_sorted(&mut b);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut c, &mut a]);
-            list.add_sorted(&mut b);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut b, &mut a]);
-            list.add_sorted(&mut c);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut d, &mut b, &mut a]);
-            list.add_sorted(&mut c);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            list.add_sorted(&mut d);
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31, 99].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            list.add_sorted(&mut d);
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([99, 31, 7, 5].to_vec(), items);
-        }
-    }
-
-    #[test]
-    fn drain_and_collect() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-
-            let taken_items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7, 31].to_vec(), taken_items);
-        }
-    }
-
-    #[test]
-    fn peek_last() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-
-            let last = list.peek_last();
-            assert_eq!(31, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert_eq!(7, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert_eq!(5, **last.unwrap());
-            list.remove_last();
-
-            let last = list.peek_last();
-            assert!(last.is_none());
-        }
-    }
-
-    #[test]
-    fn remove_first() {
-        unsafe {
-            // We iterate forward and backwards through the manipulated lists
-            // to make sure pointers in both directions are still ok.
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([7, 31].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([31, 7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert!(items.is_empty());
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_first().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert!(items.is_empty());
-        }
-    }
-
-    #[test]
-    fn remove_last() {
-        unsafe {
-            // We iterate forward and backwards through the manipulated lists
-            // to make sure pointers in both directions are still ok.
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5, 7].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([7, 5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut b, &mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(!list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert_eq!([5].to_vec(), items);
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_list(list);
-            assert!(items.is_empty());
-
-            let mut list = LinkedList::new();
-            add_nodes(&mut list, &mut [&mut a]);
-            let removed = list.remove_last().unwrap();
-            assert_clean(removed);
-            assert!(list.is_empty());
-            let items: Vec<i32> = collect_reverse_list(list);
-            assert!(items.is_empty());
-        }
-    }
-
-    #[test]
-    fn remove_by_address() {
-        unsafe {
-            let mut a = ListNode::new(5);
-            let mut b = ListNode::new(7);
-            let mut c = ListNode::new(31);
-
-            {
-                // Remove first
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut c).into()), b.next);
-                assert_eq!(Some((&mut b).into()), c.prev);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([7, 31].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut c).into()), b.next);
-                assert_eq!(Some((&mut b).into()), c.prev);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([31, 7].to_vec(), items);
-            }
-
-            {
-                // Remove middle
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut c).into()), a.next);
-                assert_eq!(Some((&mut a).into()), c.prev);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5, 31].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut c).into()), a.next);
-                assert_eq!(Some((&mut a).into()), c.prev);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([31, 5].to_vec(), items);
-            }
-
-            {
-                // Remove last
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut c));
-                assert_clean((&mut c).into());
-                assert!(b.next.is_none());
-                assert_eq!(Some((&mut b).into()), list.tail);
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5, 7].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut c, &mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut c));
-                assert_clean((&mut c).into());
-                assert!(b.next.is_none());
-                assert_eq!(Some((&mut b).into()), list.tail);
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([7, 5].to_vec(), items);
-            }
-
-            {
-                // Remove first of two
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut b).into()), list.tail);
-                assert!(b.next.is_none());
-                assert!(b.prev.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([7].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                // a should be no longer there and can't be removed twice
-                assert_eq!(false, list.remove(&mut a));
-                assert_eq!(Some((&mut b).into()), list.head);
-                assert_eq!(Some((&mut b).into()), list.tail);
-                assert!(b.next.is_none());
-                assert!(b.prev.is_none());
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([7].to_vec(), items);
-            }
-
-            {
-                // Remove last of two
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut a).into()), list.head);
-                assert_eq!(Some((&mut a).into()), list.tail);
-                assert!(a.next.is_none());
-                assert!(a.prev.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert_eq!([5].to_vec(), items);
-
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut b, &mut a]);
-                assert_eq!(true, list.remove(&mut b));
-                assert_clean((&mut b).into());
-                assert_eq!(Some((&mut a).into()), list.head);
-                assert_eq!(Some((&mut a).into()), list.tail);
-                assert!(a.next.is_none());
-                assert!(a.prev.is_none());
-                let items: Vec<i32> = collect_reverse_list(list);
-                assert_eq!([5].to_vec(), items);
-            }
-
-            {
-                // Remove last item
-                let mut list = LinkedList::new();
-                add_nodes(&mut list, &mut [&mut a]);
-                assert_eq!(true, list.remove(&mut a));
-                assert_clean((&mut a).into());
-                assert!(list.head.is_none());
-                assert!(list.tail.is_none());
-                let items: Vec<i32> = collect_list(list);
-                assert!(items.is_empty());
-            }
-
-            {
-                // Remove missing
-                let mut list = LinkedList::new();
-                list.add_front(&mut b);
-                list.add_front(&mut a);
-                assert_eq!(false, list.remove(&mut c));
-            }
-        }
-    }
-}

tokio-util/src/sync/mod.rs

@@ -3,10 +3,8 @@
 mod cancellation_token;
 pub use cancellation_token::{guard::DropGuard, CancellationToken, WaitForCancellationFuture};
 
-mod intrusive_double_linked_list;
-
 mod mpsc;
-pub use mpsc::PollSender;
+pub use mpsc::{PollSendError, PollSender};
 
 mod poll_semaphore;
 pub use poll_semaphore::PollSemaphore;

tokio-util/src/sync/mpsc.rs

@@ -1,221 +1,283 @@
-use futures_core::ready;
 use futures_sink::Sink;
 use std::pin::Pin;
-use std::sync::Arc;
 use std::task::{Context, Poll};
-use tokio::sync::mpsc::{error::SendError, Sender};
+use std::{fmt, mem};
+use tokio::sync::mpsc::OwnedPermit;
+use tokio::sync::mpsc::Sender;
 
 use super::ReusableBoxFuture;
 
-// This implementation was chosen over something based on permits because to get a
-// `tokio::sync::mpsc::Permit` out of the `inner` future, you must transmute the
-// lifetime on the permit to `'static`.
+/// Error returned by the `PollSender` when the channel is closed.
+#[derive(Debug)]
+pub struct PollSendError<T>(Option<T>);
+
+impl<T> PollSendError<T> {
+    /// Consumes the stored value, if any.
+    ///
+    /// If this error was encountered when calling `start_send`/`send_item`, this will be the item
+    /// that the caller attempted to send.  Otherwise, it will be `None`.
+    pub fn into_inner(self) -> Option<T> {
+        self.0
+    }
+}
+
+impl<T> fmt::Display for PollSendError<T> {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(fmt, "channel closed")
+    }
+}
+
+impl<T: fmt::Debug> std::error::Error for PollSendError<T> {}
+
+#[derive(Debug)]
+enum State<T> {
+    Idle(Sender<T>),
+    Acquiring,
+    ReadyToSend(OwnedPermit<T>),
+    Closed,
+}
 
 /// A wrapper around [`mpsc::Sender`] that can be polled.
 ///
 /// [`mpsc::Sender`]: tokio::sync::mpsc::Sender
 #[derive(Debug)]
 pub struct PollSender<T> {
-    /// is none if closed
-    sender: Option<Arc<Sender<T>>>,
-    is_sending: bool,
-    inner: ReusableBoxFuture<Result<(), SendError<T>>>,
+    sender: Option<Sender<T>>,
+    state: State<T>,
+    acquire: ReusableBoxFuture<'static, Result<OwnedPermit<T>, PollSendError<T>>>,
 }
 
-// By reusing the same async fn for both Some and None, we make sure every
-// future passed to ReusableBoxFuture has the same underlying type, and hence
-// the same size and alignment.
-async fn make_future<T>(data: Option<(Arc<Sender<T>>, T)>) -> Result<(), SendError<T>> {
+// Creates a future for acquiring a permit from the underlying channel.  This is used to ensure
+// there's capacity for a send to complete.
+//
+// By reusing the same async fn for both `Some` and `None`, we make sure every future passed to
+// ReusableBoxFuture has the same underlying type, and hence the same size and alignment.
+async fn make_acquire_future<T>(
+    data: Option<Sender<T>>,
+) -> Result<OwnedPermit<T>, PollSendError<T>> {
     match data {
-        Some((sender, value)) => sender.send(value).await,
-        None => unreachable!(
-            "This future should not be pollable, as is_sending should be set to false."
-        ),
+        Some(sender) => sender
+            .reserve_owned()
+            .await
+            .map_err(|_| PollSendError(None)),
+        None => unreachable!("this future should not be pollable in this state"),
     }
 }
 
 impl<T: Send + 'static> PollSender<T> {
-    /// Create a new `PollSender`.
+    /// Creates a new `PollSender`.
     pub fn new(sender: Sender<T>) -> Self {
         Self {
-            sender: Some(Arc::new(sender)),
-            is_sending: false,
-            inner: ReusableBoxFuture::new(make_future(None)),
+            sender: Some(sender.clone()),
+            state: State::Idle(sender),
+            acquire: ReusableBoxFuture::new(make_acquire_future(None)),
         }
     }
 
-    /// Start sending a new item.
+    fn take_state(&mut self) -> State<T> {
+        mem::replace(&mut self.state, State::Closed)
+    }
+
+    /// Attempts to prepare the sender to receive a value.
     ///
-    /// This method panics if a send is currently in progress. To ensure that no
-    /// send is in progress, call `poll_send_done` first until it returns
-    /// `Poll::Ready`.
+    /// This method must be called and return `Poll::Ready(Ok(()))` prior to each call to
+    /// `send_item`.
     ///
-    /// If this method returns an error, that indicates that the channel is
-    /// closed. Note that this method is not guaranteed to return an error if
-    /// the channel is closed, but in that case the error would be reported by
-    /// the first call to `poll_send_done`.
-    pub fn start_send(&mut self, value: T) -> Result<(), SendError<T>> {
-        if self.is_sending {
-            panic!("start_send called while not ready.");
-        }
-        match self.sender.clone() {
-            Some(sender) => {
-                self.inner.set(make_future(Some((sender, value))));
-                self.is_sending = true;
-                Ok(())
+    /// This method returns `Poll::Ready` once the underlying channel is ready to receive a value,
+    /// by reserving a slot in the channel for the item to be sent. If this method returns
+    /// `Poll::Pending`, the current task is registered to be notified (via
+    /// `cx.waker().wake_by_ref()`) when `poll_reserve` should be called again.
+    ///
+    /// # Errors
+    ///
+    /// If the channel is closed, an error will be returned.  This is a permanent state.
+    pub fn poll_reserve(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), PollSendError<T>>> {
+        loop {
+            let (result, next_state) = match self.take_state() {
+                State::Idle(sender) => {
+                    // Start trying to acquire a permit to reserve a slot for our send, and
+                    // immediately loop back around to poll it the first time.
+                    self.acquire.set(make_acquire_future(Some(sender)));
+                    (None, State::Acquiring)
+                }
+                State::Acquiring => match self.acquire.poll(cx) {
+                    // Channel has capacity.
+                    Poll::Ready(Ok(permit)) => {
+                        (Some(Poll::Ready(Ok(()))), State::ReadyToSend(permit))
+                    }
+                    // Channel is closed.
+                    Poll::Ready(Err(e)) => (Some(Poll::Ready(Err(e))), State::Closed),
+                    // Channel doesn't have capacity yet, so we need to wait.
+                    Poll::Pending => (Some(Poll::Pending), State::Acquiring),
+                },
+                // We're closed, either by choice or because the underlying sender was closed.
+                s @ State::Closed => (Some(Poll::Ready(Err(PollSendError(None)))), s),
+                // We're already ready to send an item.
+                s @ State::ReadyToSend(_) => (Some(Poll::Ready(Ok(()))), s),
+            };
+
+            self.state = next_state;
+            if let Some(result) = result {
+                return result;
             }
-            None => Err(SendError(value)),
         }
     }
 
-    /// If a send is in progress, poll for its completion. If no send is in progress,
-    /// this method returns `Poll::Ready(Ok(()))`.
+    /// Sends an item to the channel.
     ///
-    /// This method can return the following values:
+    /// Before calling `send_item`, `poll_reserve` must be called with a successful return
+    /// value of `Poll::Ready(Ok(()))`.
     ///
-    ///  - `Poll::Ready(Ok(()))` if the in-progress send has been completed, or there is
-    ///    no send in progress (even if the channel is closed).
-    ///  - `Poll::Ready(Err(err))` if the in-progress send failed because the channel has
-    ///    been closed.
-    ///  - `Poll::Pending` if a send is in progress, but it could not complete now.
+    /// # Errors
     ///
-    /// When this method returns `Poll::Pending`, the current task is scheduled
-    /// to receive a wakeup when the message is sent, or when the entire channel
-    /// is closed (but not if just this sender is closed by
-    /// `close_this_sender`). Note that on multiple calls to `poll_send_done`,
-    /// only the `Waker` from the `Context` passed to the most recent call is
-    /// scheduled to receive a wakeup.
+    /// If the channel is closed, an error will be returned.  This is a permanent state.
     ///
-    /// If this method returns `Poll::Ready`, then `start_send` is guaranteed to
-    /// not panic.
-    pub fn poll_send_done(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), SendError<T>>> {
-        if !self.is_sending {
-            return Poll::Ready(Ok(()));
-        }
+    /// # Panics
+    ///
+    /// If `poll_reserve` was not successfully called prior to calling `send_item`, then this method
+    /// will panic.
+    pub fn send_item(&mut self, value: T) -> Result<(), PollSendError<T>> {
+        let (result, next_state) = match self.take_state() {
+            State::Idle(_) | State::Acquiring => {
+                panic!("`send_item` called without first calling `poll_reserve`")
+            }
+            // We have a permit to send our item, so go ahead, which gets us our sender back.
+            State::ReadyToSend(permit) => (Ok(()), State::Idle(permit.send(value))),
+            // We're closed, either by choice or because the underlying sender was closed.
+            State::Closed => (Err(PollSendError(Some(value))), State::Closed),
+        };
 
-        let result = self.inner.poll(cx);
-        if result.is_ready() {
-            self.is_sending = false;
-        }
-        if let Poll::Ready(Err(_)) = &result {
-            self.sender = None;
-        }
+        // Handle deferred closing if `close` was called between `poll_reserve` and `send_item`.
+        self.state = if self.sender.is_some() {
+            next_state
+        } else {
+            State::Closed
+        };
         result
     }
 
-    /// Check whether the channel is ready to send more messages now.
+    /// Checks whether this sender is been closed.
     ///
-    /// If this method returns `true`, then `start_send` is guaranteed to not
-    /// panic.
-    ///
-    /// If the channel is closed, this method returns `true`.
-    pub fn is_ready(&self) -> bool {
-        !self.is_sending
-    }
-
-    /// Check whether the channel has been closed.
+    /// The underlying channel that this sender was wrapping may still be open.
     pub fn is_closed(&self) -> bool {
-        match &self.sender {
-            Some(sender) => sender.is_closed(),
-            None => true,
-        }
+        matches!(self.state, State::Closed) || self.sender.is_none()
     }
 
-    /// Clone the underlying `Sender`.
+    /// Gets a reference to the `Sender` of the underlying channel.
     ///
-    /// If this method returns `None`, then the channel is closed. (But it is
-    /// not guaranteed to return `None` if the channel is closed.)
-    pub fn clone_inner(&self) -> Option<Sender<T>> {
-        self.sender.as_ref().map(|sender| (&**sender).clone())
+    /// If `PollSender` has been closed, `None` is returned. The underlying channel that this sender
+    /// was wrapping may still be open.
+    pub fn get_ref(&self) -> Option<&Sender<T>> {
+        self.sender.as_ref()
     }
 
-    /// Access the underlying `Sender`.
+    /// Closes this sender.
     ///
-    /// If this method returns `None`, then the channel is closed. (But it is
-    /// not guaranteed to return `None` if the channel is closed.)
-    pub fn inner_ref(&self) -> Option<&Sender<T>> {
-        self.sender.as_deref()
-    }
-
-    // This operation is supported because it is required by the Sink trait.
-    /// Close this sender. No more messages can be sent from this sender.
+    /// No more messages will be able to be sent from this sender, but the underlying channel will
+    /// remain open until all senders have dropped, or until the [`Receiver`] closes the channel.
     ///
-    /// Note that this only closes the channel from the view-point of this
-    /// sender. The channel remains open until all senders have gone away, or
-    /// until the [`Receiver`] closes the channel.
-    ///
-    /// If there is a send in progress when this method is called, that send is
-    /// unaffected by this operation, and `poll_send_done` can still be called
-    /// to complete that send.
+    /// If a slot was previously reserved by calling `poll_reserve`, then a final call can be made
+    /// to `send_item` in order to consume the reserved slot.  After that, no further sends will be
+    /// possible.  If you do not intend to send another item, you can release the reserved slot back
+    /// to the underlying sender by calling [`abort_send`].
     ///
+    /// [`abort_send`]: crate::sync::PollSender::abort_send
     /// [`Receiver`]: tokio::sync::mpsc::Receiver
-    pub fn close_this_sender(&mut self) {
+    pub fn close(&mut self) {
+        // Mark ourselves officially closed by dropping our main sender.
         self.sender = None;
+
+        // If we're already idle, closed, or we haven't yet reserved a slot, we can quickly
+        // transition to the closed state.  Otherwise, leave the existing permit in place for the
+        // caller if they want to complete the send.
+        match self.state {
+            State::Idle(_) => self.state = State::Closed,
+            State::Acquiring => {
+                self.acquire.set(make_acquire_future(None));
+                self.state = State::Closed;
+            }
+            _ => {}
+        }
     }
 
-    /// Abort the current in-progress send, if any.
+    /// Aborts the current in-progress send, if any.
     ///
-    /// Returns `true` if a send was aborted.
+    /// Returns `true` if a send was aborted.  If the sender was closed prior to calling
+    /// `abort_send`, then the sender will remain in the closed state, otherwise the sender will be
+    /// ready to attempt another send.
     pub fn abort_send(&mut self) -> bool {
-        if self.is_sending {
-            self.inner.set(make_future(None));
-            self.is_sending = false;
-            true
-        } else {
-            false
-        }
+        // We may have been closed in the meantime, after a call to `poll_reserve` already
+        // succeeded.  We'll check if `self.sender` is `None` to see if we should transition to the
+        // closed state when we actually abort a send, rather than resetting ourselves back to idle.
+
+        let (result, next_state) = match self.take_state() {
+            // We're currently trying to reserve a slot to send into.
+            State::Acquiring => {
+                // Replacing the future drops the in-flight one.
+                self.acquire.set(make_acquire_future(None));
+
+                // If we haven't closed yet, we have to clone our stored sender since we have no way
+                // to get it back from the acquire future we just dropped.
+                let state = match self.sender.clone() {
+                    Some(sender) => State::Idle(sender),
+                    None => State::Closed,
+                };
+                (true, state)
+            }
+            // We got the permit.  If we haven't closed yet, get the sender back.
+            State::ReadyToSend(permit) => {
+                let state = if self.sender.is_some() {
+                    State::Idle(permit.release())
+                } else {
+                    State::Closed
+                };
+                (true, state)
+            }
+            s => (false, s),
+        };
+
+        self.state = next_state;
+        result
     }
 }
 
 impl<T> Clone for PollSender<T> {
-    /// Clones this `PollSender`. The resulting clone will not have any
-    /// in-progress send operations, even if the current `PollSender` does.
+    /// Clones this `PollSender`.
+    ///
+    /// The resulting `PollSender` will have an initial state identical to calling `PollSender::new`.
     fn clone(&self) -> PollSender<T> {
+        let (sender, state) = match self.sender.clone() {
+            Some(sender) => (Some(sender.clone()), State::Idle(sender)),
+            None => (None, State::Closed),
+        };
+
         Self {
-            sender: self.sender.clone(),
-            is_sending: false,
-            inner: ReusableBoxFuture::new(async { unreachable!() }),
+            sender,
+            state,
+            // We don't use `make_acquire_future` here because our relaxed bounds on `T` are not
+            // compatible with the transitive bounds required by `Sender<T>`.
+            acquire: ReusableBoxFuture::new(async { unreachable!() }),
         }
     }
 }
 
 impl<T: Send + 'static> Sink<T> for PollSender<T> {
-    type Error = SendError<T>;
+    type Error = PollSendError<T>;
 
-    /// This is equivalent to calling [`poll_send_done`].
-    ///
-    /// [`poll_send_done`]: PollSender::poll_send_done
     fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        Pin::into_inner(self).poll_send_done(cx)
+        Pin::into_inner(self).poll_reserve(cx)
     }
 
-    /// This is equivalent to calling [`poll_send_done`].
-    ///
-    /// [`poll_send_done`]: PollSender::poll_send_done
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        Pin::into_inner(self).poll_send_done(cx)
+    fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
     }
 
-    /// This is equivalent to calling [`start_send`].
-    ///
-    /// [`start_send`]: PollSender::start_send
     fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
-        Pin::into_inner(self).start_send(item)
+        Pin::into_inner(self).send_item(item)
     }
 
-    /// This method will first flush the `PollSender`, and then close it by
-    /// calling [`close_this_sender`].
-    ///
-    /// If a send fails while flushing because the [`Receiver`] has gone away,
-    /// then this function returns an error. The channel is still successfully
-    /// closed in this situation.
-    ///
-    /// [`close_this_sender`]: PollSender::close_this_sender
-    /// [`Receiver`]: tokio::sync::mpsc::Receiver
-    fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        ready!(self.as_mut().poll_flush(cx))?;
-
-        Pin::into_inner(self).close_this_sender();
+    fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Pin::into_inner(self).close();
         Poll::Ready(Ok(()))
     }
 }

tokio-util/src/sync/poll_semaphore.rs

@@ -12,7 +12,7 @@ use super::ReusableBoxFuture;
 /// [`Semaphore`]: tokio::sync::Semaphore
 pub struct PollSemaphore {
     semaphore: Arc<Semaphore>,
-    permit_fut: Option<ReusableBoxFuture<Result<OwnedSemaphorePermit, AcquireError>>>,
+    permit_fut: Option<ReusableBoxFuture<'static, Result<OwnedSemaphorePermit, AcquireError>>>,
 }
 
 impl PollSemaphore {

tokio-util/src/sync/reusable_box.rs

@@ -1,33 +1,29 @@
 use std::alloc::Layout;
+use std::fmt;
 use std::future::Future;
-use std::panic::AssertUnwindSafe;
+use std::marker::PhantomData;
+use std::mem::{self, ManuallyDrop};
 use std::pin::Pin;
-use std::ptr::{self, NonNull};
+use std::ptr;
 use std::task::{Context, Poll};
-use std::{fmt, panic};
 
-/// A reusable `Pin<Box<dyn Future<Output = T> + Send>>`.
+/// A reusable `Pin<Box<dyn Future<Output = T> + Send + 'a>>`.
 ///
 /// This type lets you replace the future stored in the box without
 /// reallocating when the size and alignment permits this.
-pub struct ReusableBoxFuture<T> {
-    boxed: NonNull<dyn Future<Output = T> + Send>,
+pub struct ReusableBoxFuture<'a, T> {
+    boxed: Pin<Box<dyn Future<Output = T> + Send + 'a>>,
 }
 
-impl<T> ReusableBoxFuture<T> {
+impl<'a, T> ReusableBoxFuture<'a, T> {
     /// Create a new `ReusableBoxFuture<T>` containing the provided future.
     pub fn new<F>(future: F) -> Self
     where
-        F: Future<Output = T> + Send + 'static,
+        F: Future<Output = T> + Send + 'a,
     {
-        let boxed: Box<dyn Future<Output = T> + Send> = Box::new(future);
-
-        let boxed = Box::into_raw(boxed);
-
-        // SAFETY: Box::into_raw does not return null pointers.
-        let boxed = unsafe { NonNull::new_unchecked(boxed) };
-
-        Self { boxed }
+        Self {
+            boxed: Box::pin(future),
+        }
     }
 
     /// Replace the future currently stored in this box.
@@ -36,7 +32,7 @@ impl<T> ReusableBoxFuture<T> {
     /// different from the layout of the currently stored future.
     pub fn set<F>(&mut self, future: F)
     where
-        F: Future<Output = T> + Send + 'static,
+        F: Future<Output = T> + Send + 'a,
     {
         if let Err(future) = self.try_set(future) {
             *self = Self::new(future);
@@ -50,64 +46,31 @@ impl<T> ReusableBoxFuture<T> {
     /// future.
     pub fn try_set<F>(&mut self, future: F) -> Result<(), F>
     where
-        F: Future<Output = T> + Send + 'static,
+        F: Future<Output = T> + Send + 'a,
     {
-        // SAFETY: The pointer is not dangling.
-        let self_layout = {
-            let dyn_future: &(dyn Future<Output = T> + Send) = unsafe { self.boxed.as_ref() };
-            Layout::for_value(dyn_future)
-        };
-
-        if Layout::new::<F>() == self_layout {
-            // SAFETY: We just checked that the layout of F is correct.
-            unsafe {
-                self.set_same_layout(future);
-            }
-
-            Ok(())
-        } else {
-            Err(future)
+        // If we try to inline the contents of this function, the type checker complains because
+        // the bound `T: 'a` is not satisfied in the call to `pending()`. But by putting it in an
+        // inner function that doesn't have `T` as a generic parameter, we implicitly get the bound
+        // `F::Output: 'a` transitively through `F: 'a`, allowing us to call `pending()`.
+        #[inline(always)]
+        fn real_try_set<'a, F>(
+            this: &mut ReusableBoxFuture<'a, F::Output>,
+            future: F,
+        ) -> Result<(), F>
+        where
+            F: Future + Send + 'a,
+        {
+            // future::Pending<T> is a ZST so this never allocates.
+            let boxed = mem::replace(&mut this.boxed, Box::pin(Pending(PhantomData)));
+            reuse_pin_box(boxed, future, |boxed| this.boxed = Pin::from(boxed))
         }
-    }
 
-    /// Set the current future.
-    ///
-    /// # Safety
-    ///
-    /// This function requires that the layout of the provided future is the
-    /// same as `self.layout`.
-    unsafe fn set_same_layout<F>(&mut self, future: F)
-    where
-        F: Future<Output = T> + Send + 'static,
-    {
-        // Drop the existing future, catching any panics.
-        let result = panic::catch_unwind(AssertUnwindSafe(|| {
-            ptr::drop_in_place(self.boxed.as_ptr());
-        }));
-
-        // Overwrite the future behind the pointer. This is safe because the
-        // allocation was allocated with the same size and alignment as the type F.
-        let self_ptr: *mut F = self.boxed.as_ptr() as *mut F;
-        ptr::write(self_ptr, future);
-
-        // Update the vtable of self.boxed. The pointer is not null because we
-        // just got it from self.boxed, which is not null.
-        self.boxed = NonNull::new_unchecked(self_ptr);
-
-        // If the old future's destructor panicked, resume unwinding.
-        match result {
-            Ok(()) => {}
-            Err(payload) => {
-                panic::resume_unwind(payload);
-            }
-        }
+        real_try_set(self, future)
     }
 
     /// Get a pinned reference to the underlying future.
     pub fn get_pin(&mut self) -> Pin<&mut (dyn Future<Output = T> + Send)> {
-        // SAFETY: The user of this box cannot move the box, and we do not move it
-        // either.
-        unsafe { Pin::new_unchecked(self.boxed.as_mut()) }
+        self.boxed.as_mut()
     }
 
     /// Poll the future stored inside this box.
@@ -116,7 +79,7 @@ impl<T> ReusableBoxFuture<T> {
     }
 }
 
-impl<T> Future for ReusableBoxFuture<T> {
+impl<T> Future for ReusableBoxFuture<'_, T> {
     type Output = T;
 
     /// Poll the future stored inside this box.
@@ -125,27 +88,84 @@ impl<T> Future for ReusableBoxFuture<T> {
     }
 }
 
-// The future stored inside ReusableBoxFuture<T> must be Send.
-unsafe impl<T> Send for ReusableBoxFuture<T> {}
-
 // The only method called on self.boxed is poll, which takes &mut self, so this
 // struct being Sync does not permit any invalid access to the Future, even if
 // the future is not Sync.
-unsafe impl<T> Sync for ReusableBoxFuture<T> {}
+unsafe impl<T> Sync for ReusableBoxFuture<'_, T> {}
 
-// Just like a Pin<Box<dyn Future>> is always Unpin, so is this type.
-impl<T> Unpin for ReusableBoxFuture<T> {}
-
-impl<T> Drop for ReusableBoxFuture<T> {
-    fn drop(&mut self) {
-        unsafe {
-            drop(Box::from_raw(self.boxed.as_ptr()));
-        }
-    }
-}
-
-impl<T> fmt::Debug for ReusableBoxFuture<T> {
+impl<T> fmt::Debug for ReusableBoxFuture<'_, T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("ReusableBoxFuture").finish()
     }
 }
+
+fn reuse_pin_box<T: ?Sized, U, O, F>(boxed: Pin<Box<T>>, new_value: U, callback: F) -> Result<O, U>
+where
+    F: FnOnce(Box<U>) -> O,
+{
+    let layout = Layout::for_value::<T>(&*boxed);
+    if layout != Layout::new::<U>() {
+        return Err(new_value);
+    }
+
+    // SAFETY: We don't ever construct a non-pinned reference to the old `T` from now on, and we
+    // always drop the `T`.
+    let raw: *mut T = Box::into_raw(unsafe { Pin::into_inner_unchecked(boxed) });
+
+    // When dropping the old value panics, we still want to call `callback` — so move the rest of
+    // the code into a guard type.
+    let guard = CallOnDrop::new(|| {
+        let raw: *mut U = raw.cast::<U>();
+        unsafe { raw.write(new_value) };
+
+        // SAFETY:
+        // - `T` and `U` have the same layout.
+        // - `raw` comes from a `Box` that uses the same allocator as this one.
+        // - `raw` points to a valid instance of `U` (we just wrote it in).
+        let boxed = unsafe { Box::from_raw(raw) };
+
+        callback(boxed)
+    });
+
+    // Drop the old value.
+    unsafe { ptr::drop_in_place(raw) };
+
+    // Run the rest of the code.
+    Ok(guard.call())
+}
+
+struct CallOnDrop<O, F: FnOnce() -> O> {
+    f: ManuallyDrop<F>,
+}
+
+impl<O, F: FnOnce() -> O> CallOnDrop<O, F> {
+    fn new(f: F) -> Self {
+        let f = ManuallyDrop::new(f);
+        Self { f }
+    }
+    fn call(self) -> O {
+        let mut this = ManuallyDrop::new(self);
+        let f = unsafe { ManuallyDrop::take(&mut this.f) };
+        f()
+    }
+}
+
+impl<O, F: FnOnce() -> O> Drop for CallOnDrop<O, F> {
+    fn drop(&mut self) {
+        let f = unsafe { ManuallyDrop::take(&mut self.f) };
+        f();
+    }
+}
+
+/// The same as `std::future::Pending<T>`; we can't use that type directly because on rustc
+/// versions <1.60 it didn't unconditionally implement `Send`.
+// FIXME: use `std::future::Pending<T>` once the MSRV is >=1.60
+struct Pending<T>(PhantomData<fn() -> T>);
+
+impl<T> Future for Pending<T> {
+    type Output = T;
+
+    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
+        Poll::Pending
+    }
+}

tokio-util/src/task/join_map.rs

@@ -0,0 +1,810 @@
+use hashbrown::hash_map::RawEntryMut;
+use hashbrown::HashMap;
+use std::borrow::Borrow;
+use std::collections::hash_map::RandomState;
+use std::fmt;
+use std::future::Future;
+use std::hash::{BuildHasher, Hash, Hasher};
+use tokio::runtime::Handle;
+use tokio::task::{AbortHandle, Id, JoinError, JoinSet, LocalSet};
+
+/// A collection of tasks spawned on a Tokio runtime, associated with hash map
+/// keys.
+///
+/// This type is very similar to the [`JoinSet`] type in `tokio::task`, with the
+/// addition of a  set of keys associated with each task. These keys allow
+/// [cancelling a task][abort] or [multiple tasks][abort_matching] in the
+/// `JoinMap` based on   their keys, or [test whether a task corresponding to a
+/// given key exists][contains] in the `JoinMap`.
+///
+/// In addition, when tasks in the `JoinMap` complete, they will return the
+/// associated key along with the value returned by the task, if any.
+///
+/// A `JoinMap` can be used to await the completion of some or all of the tasks
+/// in the map. The map is not ordered, and the tasks will be returned in the
+/// order they complete.
+///
+/// All of the tasks must have the same return type `V`.
+///
+/// When the `JoinMap` is dropped, all tasks in the `JoinMap` are immediately aborted.
+///
+/// **Note**: This type depends on Tokio's [unstable API][unstable]. See [the
+/// documentation on unstable features][unstable] for details on how to enable
+/// Tokio's unstable features.
+///
+/// # Examples
+///
+/// Spawn multiple tasks and wait for them:
+///
+/// ```
+/// use tokio_util::task::JoinMap;
+///
+/// #[tokio::main]
+/// async fn main() {
+///     let mut map = JoinMap::new();
+///
+///     for i in 0..10 {
+///         // Spawn a task on the `JoinMap` with `i` as its key.
+///         map.spawn(i, async move { /* ... */ });
+///     }
+///
+///     let mut seen = [false; 10];
+///
+///     // When a task completes, `join_one` returns the task's key along
+///     // with its output.
+///     while let Some((key, res)) = map.join_one().await {
+///         seen[key] = true;
+///         assert!(res.is_ok(), "task {} completed successfully!", key);
+///     }
+///
+///     for i in 0..10 {
+///         assert!(seen[i]);
+///     }
+/// }
+/// ```
+///
+/// Cancel tasks based on their keys:
+///
+/// ```
+/// use tokio_util::task::JoinMap;
+///
+/// #[tokio::main]
+/// async fn main() {
+///     let mut map = JoinMap::new();
+///
+///     map.spawn("hello world", async move { /* ... */ });
+///     map.spawn("goodbye world", async move { /* ... */});
+///
+///     // Look up the "goodbye world" task in the map and abort it.
+///     let aborted = map.abort("goodbye world");
+///
+///     // `JoinMap::abort` returns `true` if a task existed for the
+///     // provided key.
+///     assert!(aborted);
+///
+///     while let Some((key, res)) = map.join_one().await {
+///         if key == "goodbye world" {
+///             // The aborted task should complete with a cancelled `JoinError`.
+///             assert!(res.unwrap_err().is_cancelled());
+///         } else {
+///             // Other tasks should complete normally.
+///             assert!(res.is_ok());
+///         }
+///     }
+/// }
+/// ```
+///
+/// [`JoinSet`]: tokio::task::JoinSet
+/// [unstable]: tokio#unstable-features
+/// [abort]: fn@Self::abort
+/// [abort_matching]: fn@Self::abort_matching
+/// [contains]: fn@Self::contains_key
+#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))]
+pub struct JoinMap<K, V, S = RandomState> {
+    /// A map of the [`AbortHandle`]s of the tasks spawned on this `JoinMap`,
+    /// indexed by their keys and task IDs.
+    ///
+    /// The [`Key`] type contains both the task's `K`-typed key provided when
+    /// spawning tasks, and the task's IDs. The IDs are stored here to resolve
+    /// hash collisions when looking up tasks based on their pre-computed hash
+    /// (as stored in the `hashes_by_task` map).
+    tasks_by_key: HashMap<Key<K>, AbortHandle, S>,
+
+    /// A map from task IDs to the hash of the key associated with that task.
+    ///
+    /// This map is used to perform reverse lookups of tasks in the
+    /// `tasks_by_key` map based on their task IDs. When a task terminates, the
+    /// ID is provided to us by the `JoinSet`, so we can look up the hash value
+    /// of that task's key, and then remove it from the `tasks_by_key` map using
+    /// the raw hash code, resolving collisions by comparing task IDs.
+    hashes_by_task: HashMap<Id, u64, S>,
+
+    /// The [`JoinSet`] that awaits the completion of tasks spawned on this
+    /// `JoinMap`.
+    tasks: JoinSet<V>,
+}
+
+/// A [`JoinMap`] key.
+///
+/// This holds both a `K`-typed key (the actual key as seen by the user), _and_
+/// a task ID, so that hash collisions between `K`-typed keys can be resolved
+/// using either `K`'s `Eq` impl *or* by checking the task IDs.
+///
+/// This allows looking up a task using either an actual key (such as when the
+/// user queries the map with a key), *or* using a task ID and a hash (such as
+/// when removing completed tasks from the map).
+#[derive(Debug)]
+struct Key<K> {
+    key: K,
+    id: Id,
+}
+
+impl<K, V> JoinMap<K, V> {
+    /// Creates a new empty `JoinMap`.
+    ///
+    /// The `JoinMap` is initially created with a capacity of 0, so it will not
+    /// allocate until a task is first spawned on it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    /// let map: JoinMap<&str, i32> = JoinMap::new();
+    /// ```
+    #[inline]
+    #[must_use]
+    pub fn new() -> Self {
+        Self::with_hasher(RandomState::new())
+    }
+
+    /// Creates an empty `JoinMap` with the specified capacity.
+    ///
+    /// The `JoinMap` will be able to hold at least `capacity` tasks without
+    /// reallocating.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    /// let map: JoinMap<&str, i32> = JoinMap::with_capacity(10);
+    /// ```
+    #[inline]
+    #[must_use]
+    pub fn with_capacity(capacity: usize) -> Self {
+        JoinMap::with_capacity_and_hasher(capacity, Default::default())
+    }
+}
+
+impl<K, V, S: Clone> JoinMap<K, V, S> {
+    /// Creates an empty `JoinMap` which will use the given hash builder to hash
+    /// keys.
+    ///
+    /// The created map has the default initial capacity.
+    ///
+    /// Warning: `hash_builder` is normally randomly generated, and
+    /// is designed to allow `JoinMap` to be resistant to attacks that
+    /// cause many collisions and very poor performance. Setting it
+    /// manually using this function can expose a DoS attack vector.
+    ///
+    /// The `hash_builder` passed should implement the [`BuildHasher`] trait for
+    /// the `JoinMap` to be useful, see its documentation for details.
+    #[inline]
+    #[must_use]
+    pub fn with_hasher(hash_builder: S) -> Self {
+        Self::with_capacity_and_hasher(0, hash_builder)
+    }
+
+    /// Creates an empty `JoinMap` with the specified capacity, using `hash_builder`
+    /// to hash the keys.
+    ///
+    /// The `JoinMap` will be able to hold at least `capacity` elements without
+    /// reallocating. If `capacity` is 0, the `JoinMap` will not allocate.
+    ///
+    /// Warning: `hash_builder` is normally randomly generated, and
+    /// is designed to allow HashMaps to be resistant to attacks that
+    /// cause many collisions and very poor performance. Setting it
+    /// manually using this function can expose a DoS attack vector.
+    ///
+    /// The `hash_builder` passed should implement the [`BuildHasher`] trait for
+    /// the `JoinMap`to be useful, see its documentation for details.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_util::task::JoinMap;
+    /// use std::collections::hash_map::RandomState;
+    ///
+    /// let s = RandomState::new();
+    /// let mut map = JoinMap::with_capacity_and_hasher(10, s);
+    /// map.spawn(1, async move { "hello world!" });
+    /// # }
+    /// ```
+    #[inline]
+    #[must_use]
+    pub fn with_capacity_and_hasher(capacity: usize, hash_builder: S) -> Self {
+        Self {
+            tasks_by_key: HashMap::with_capacity_and_hasher(capacity, hash_builder.clone()),
+            hashes_by_task: HashMap::with_capacity_and_hasher(capacity, hash_builder),
+            tasks: JoinSet::new(),
+        }
+    }
+
+    /// Returns the number of tasks currently in the `JoinMap`.
+    pub fn len(&self) -> usize {
+        let len = self.tasks_by_key.len();
+        debug_assert_eq!(len, self.hashes_by_task.len());
+        len
+    }
+
+    /// Returns whether the `JoinMap` is empty.
+    pub fn is_empty(&self) -> bool {
+        let empty = self.tasks_by_key.is_empty();
+        debug_assert_eq!(empty, self.hashes_by_task.is_empty());
+        empty
+    }
+
+    /// Returns the number of tasks the map can hold without reallocating.
+    ///
+    /// This number is a lower bound; the `JoinMap` might be able to hold
+    /// more, but is guaranteed to be able to hold at least this many.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// let map: JoinMap<i32, i32> = JoinMap::with_capacity(100);
+    /// assert!(map.capacity() >= 100);
+    /// ```
+    #[inline]
+    pub fn capacity(&self) -> usize {
+        let capacity = self.tasks_by_key.capacity();
+        debug_assert_eq!(capacity, self.hashes_by_task.capacity());
+        capacity
+    }
+}
+
+impl<K, V, S> JoinMap<K, V, S>
+where
+    K: Hash + Eq,
+    V: 'static,
+    S: BuildHasher,
+{
+    /// Spawn the provided task and store it in this `JoinMap` with the provided
+    /// key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// # Panics
+    ///
+    /// This method panics if called outside of a Tokio runtime.
+    ///
+    /// [`join_one`]: Self::join_one
+    #[track_caller]
+    pub fn spawn<F>(&mut self, key: K, task: F)
+    where
+        F: Future<Output = V>,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn(task);
+        self.insert(key, task)
+    }
+
+    /// Spawn the provided task on the provided runtime and store it in this
+    /// `JoinMap` with the provided key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// [`join_one`]: Self::join_one
+    #[track_caller]
+    pub fn spawn_on<F>(&mut self, key: K, task: F, handle: &Handle)
+    where
+        F: Future<Output = V>,
+        F: Send + 'static,
+        V: Send,
+    {
+        let task = self.tasks.spawn_on(task, handle);
+        self.insert(key, task);
+    }
+
+    /// Spawn the provided task on the current [`LocalSet`] and store it in this
+    /// `JoinMap` with the provided key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// # Panics
+    ///
+    /// This method panics if it is called outside of a `LocalSet`.
+    ///
+    /// [`LocalSet`]: tokio::task::LocalSet
+    /// [`join_one`]: Self::join_one
+    #[track_caller]
+    pub fn spawn_local<F>(&mut self, key: K, task: F)
+    where
+        F: Future<Output = V>,
+        F: 'static,
+    {
+        let task = self.tasks.spawn_local(task);
+        self.insert(key, task);
+    }
+
+    /// Spawn the provided task on the provided [`LocalSet`] and store it in
+    /// this `JoinMap` with the provided key.
+    ///
+    /// If a task previously existed in the `JoinMap` for this key, that task
+    /// will be cancelled and replaced with the new one. The previous task will
+    /// be removed from the `JoinMap`; a subsequent call to [`join_one`] will
+    /// *not* return a cancelled [`JoinError`] for that task.
+    ///
+    /// [`LocalSet`]: tokio::task::LocalSet
+    /// [`join_one`]: Self::join_one
+    #[track_caller]
+    pub fn spawn_local_on<F>(&mut self, key: K, task: F, local_set: &LocalSet)
+    where
+        F: Future<Output = V>,
+        F: 'static,
+    {
+        let task = self.tasks.spawn_local_on(task, local_set);
+        self.insert(key, task)
+    }
+
+    fn insert(&mut self, key: K, abort: AbortHandle) {
+        let hash = self.hash(&key);
+        let id = abort.id();
+        let map_key = Key {
+            id: id.clone(),
+            key,
+        };
+
+        // Insert the new key into the map of tasks by keys.
+        let entry = self
+            .tasks_by_key
+            .raw_entry_mut()
+            .from_hash(hash, |k| k.key == map_key.key);
+        match entry {
+            RawEntryMut::Occupied(mut occ) => {
+                // There was a previous task spawned with the same key! Cancel
+                // that task, and remove its ID from the map of hashes by task IDs.
+                let Key { id: prev_id, .. } = occ.insert_key(map_key);
+                occ.insert(abort).abort();
+                let _prev_hash = self.hashes_by_task.remove(&prev_id);
+                debug_assert_eq!(Some(hash), _prev_hash);
+            }
+            RawEntryMut::Vacant(vac) => {
+                vac.insert(map_key, abort);
+            }
+        };
+
+        // Associate the key's hash with this task's ID, for looking up tasks by ID.
+        let _prev = self.hashes_by_task.insert(id, hash);
+        debug_assert!(_prev.is_none(), "no prior task should have had the same ID");
+    }
+
+    /// Waits until one of the tasks in the map completes and returns its
+    /// output, along with the key corresponding to that task.
+    ///
+    /// Returns `None` if the map is empty.
+    ///
+    /// # Cancel Safety
+    ///
+    /// This method is cancel safe. If `join_one` is used as the event in a [`tokio::select!`]
+    /// statement and some other branch completes first, it is guaranteed that no tasks were
+    /// removed from this `JoinMap`.
+    ///
+    /// # Returns
+    ///
+    /// This function returns:
+    ///
+    ///  * `Some((key, Ok(value)))` if one of the tasks in this `JoinMap` has
+    ///    completed. The `value` is the return value of that ask, and `key` is
+    ///    the key associated with the task.
+    ///  * `Some((key, Err(err))` if one of the tasks in this JoinMap` has
+    ///    panicked or been aborted. `key` is the key associated  with the task
+    ///    that panicked or was aborted.
+    ///  * `None` if the `JoinMap` is empty.
+    ///
+    /// [`tokio::select!`]: tokio::select
+    pub async fn join_one(&mut self) -> Option<(K, Result<V, JoinError>)> {
+        let (res, id) = match self.tasks.join_one_with_id().await {
+            Some(Ok((id, output))) => (Ok(output), id),
+            Some(Err(e)) => {
+                let id = e.id();
+                (Err(e), id)
+            }
+            None => return None,
+        };
+        let key = self.remove_by_id(id)?;
+        Some((key, res))
+    }
+
+    /// Aborts all tasks and waits for them to finish shutting down.
+    ///
+    /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_one`] in
+    /// a loop until it returns `None`.
+    ///
+    /// This method ignores any panics in the tasks shutting down. When this call returns, the
+    /// `JoinMap` will be empty.
+    ///
+    /// [`abort_all`]: fn@Self::abort_all
+    /// [`join_one`]: fn@Self::join_one
+    pub async fn shutdown(&mut self) {
+        self.abort_all();
+        while self.join_one().await.is_some() {}
+    }
+
+    /// Abort the task corresponding to the provided `key`.
+    ///
+    /// If this `JoinMap` contains a task corresponding to `key`, this method
+    /// will abort that task and return `true`. Otherwise, if no task exists for
+    /// `key`, this method returns `false`.
+    ///
+    /// # Examples
+    ///
+    /// Aborting a task by key:
+    ///
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// let mut map = JoinMap::new();
+    ///
+    /// map.spawn("hello world", async move { /* ... */ });
+    /// map.spawn("goodbye world", async move { /* ... */});
+    ///
+    /// // Look up the "goodbye world" task in the map and abort it.
+    /// map.abort("goodbye world");
+    ///
+    /// while let Some((key, res)) = map.join_one().await {
+    ///     if key == "goodbye world" {
+    ///         // The aborted task should complete with a cancelled `JoinError`.
+    ///         assert!(res.unwrap_err().is_cancelled());
+    ///     } else {
+    ///         // Other tasks should complete normally.
+    ///         assert!(res.is_ok());
+    ///     }
+    /// }
+    /// # }
+    /// ```
+    ///
+    /// `abort` returns `true` if a task was aborted:
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// let mut map = JoinMap::new();
+    ///
+    /// map.spawn("hello world", async move { /* ... */ });
+    /// map.spawn("goodbye world", async move { /* ... */});
+    ///
+    /// // A task for the key "goodbye world" should exist in the map:
+    /// assert!(map.abort("goodbye world"));
+    ///
+    /// // Aborting a key that does not exist will return `false`:
+    /// assert!(!map.abort("goodbye universe"));
+    /// # }
+    /// ```
+    pub fn abort<Q: ?Sized>(&mut self, key: &Q) -> bool
+    where
+        Q: Hash + Eq,
+        K: Borrow<Q>,
+    {
+        match self.get_by_key(key) {
+            Some((_, handle)) => {
+                handle.abort();
+                true
+            }
+            None => false,
+        }
+    }
+
+    /// Aborts all tasks with keys matching `predicate`.
+    ///
+    /// `predicate` is a function called with a reference to each key in the
+    /// map. If it returns `true` for a given key, the corresponding task will
+    /// be cancelled.
+    ///
+    /// # Examples
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// # // use the current thread rt so that spawned tasks don't
+    /// # // complete in the background before they can be aborted.
+    /// # #[tokio::main(flavor = "current_thread")]
+    /// # async fn main() {
+    /// let mut map = JoinMap::new();
+    ///
+    /// map.spawn("hello world", async move {
+    ///     // ...
+    ///     # tokio::task::yield_now().await; // don't complete immediately, get aborted!
+    /// });
+    /// map.spawn("goodbye world", async move {
+    ///     // ...
+    ///     # tokio::task::yield_now().await; // don't complete immediately, get aborted!
+    /// });
+    /// map.spawn("hello san francisco", async move {
+    ///     // ...
+    ///     # tokio::task::yield_now().await; // don't complete immediately, get aborted!
+    /// });
+    /// map.spawn("goodbye universe", async move {
+    ///     // ...
+    ///     # tokio::task::yield_now().await; // don't complete immediately, get aborted!
+    /// });
+    ///
+    /// // Abort all tasks whose keys begin with "goodbye"
+    /// map.abort_matching(|key| key.starts_with("goodbye"));
+    ///
+    /// let mut seen = 0;
+    /// while let Some((key, res)) = map.join_one().await {
+    ///     seen += 1;
+    ///     if key.starts_with("goodbye") {
+    ///         // The aborted task should complete with a cancelled `JoinError`.
+    ///         assert!(res.unwrap_err().is_cancelled());
+    ///     } else {
+    ///         // Other tasks should complete normally.
+    ///         assert!(key.starts_with("hello"));
+    ///         assert!(res.is_ok());
+    ///     }
+    /// }
+    ///
+    /// // All spawned tasks should have completed.
+    /// assert_eq!(seen, 4);
+    /// # }
+    /// ```
+    pub fn abort_matching(&mut self, mut predicate: impl FnMut(&K) -> bool) {
+        // Note: this method iterates over the tasks and keys *without* removing
+        // any entries, so that the keys from aborted tasks can still be
+        // returned when calling `join_one` in the future.
+        for (Key { ref key, .. }, task) in &self.tasks_by_key {
+            if predicate(key) {
+                task.abort();
+            }
+        }
+    }
+
+    /// Returns `true` if this `JoinMap` contains a task for the provided key.
+    ///
+    /// If the task has completed, but its output hasn't yet been consumed by a
+    /// call to [`join_one`], this method will still return `true`.
+    ///
+    /// [`join_one`]: fn@Self::join_one
+    pub fn contains_key<Q: ?Sized>(&self, key: &Q) -> bool
+    where
+        Q: Hash + Eq,
+        K: Borrow<Q>,
+    {
+        self.get_by_key(key).is_some()
+    }
+
+    /// Returns `true` if this `JoinMap` contains a task with the provided
+    /// [task ID].
+    ///
+    /// If the task has completed, but its output hasn't yet been consumed by a
+    /// call to [`join_one`], this method will still return `true`.
+    ///
+    /// [`join_one`]: fn@Self::join_one
+    /// [task ID]: tokio::task::Id
+    pub fn contains_task(&self, task: &Id) -> bool {
+        self.get_by_id(task).is_some()
+    }
+
+    /// Reserves capacity for at least `additional` more tasks to be spawned
+    /// on this `JoinMap` without reallocating for the map of task keys. The
+    /// collection may reserve more space to avoid frequent reallocations.
+    ///
+    /// Note that spawning a task will still cause an allocation for the task
+    /// itself.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new allocation size overflows [`usize`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// let mut map: JoinMap<&str, i32> = JoinMap::new();
+    /// map.reserve(10);
+    /// ```
+    #[inline]
+    pub fn reserve(&mut self, additional: usize) {
+        self.tasks_by_key.reserve(additional);
+        self.hashes_by_task.reserve(additional);
+    }
+
+    /// Shrinks the capacity of the `JoinMap` as much as possible. It will drop
+    /// down as much as possible while maintaining the internal rules
+    /// and possibly leaving some space in accordance with the resize policy.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// let mut map: JoinMap<i32, i32> = JoinMap::with_capacity(100);
+    /// map.spawn(1, async move { 2 });
+    /// map.spawn(3, async move { 4 });
+    /// assert!(map.capacity() >= 100);
+    /// map.shrink_to_fit();
+    /// assert!(map.capacity() >= 2);
+    /// # }
+    /// ```
+    #[inline]
+    pub fn shrink_to_fit(&mut self) {
+        self.hashes_by_task.shrink_to_fit();
+        self.tasks_by_key.shrink_to_fit();
+    }
+
+    /// Shrinks the capacity of the map with a lower limit. It will drop
+    /// down no lower than the supplied limit while maintaining the internal rules
+    /// and possibly leaving some space in accordance with the resize policy.
+    ///
+    /// If the current capacity is less than the lower limit, this is a no-op.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_util::task::JoinMap;
+    ///
+    /// let mut map: JoinMap<i32, i32> = JoinMap::with_capacity(100);
+    /// map.spawn(1, async move { 2 });
+    /// map.spawn(3, async move { 4 });
+    /// assert!(map.capacity() >= 100);
+    /// map.shrink_to(10);
+    /// assert!(map.capacity() >= 10);
+    /// map.shrink_to(0);
+    /// assert!(map.capacity() >= 2);
+    /// # }
+    /// ```
+    #[inline]
+    pub fn shrink_to(&mut self, min_capacity: usize) {
+        self.hashes_by_task.shrink_to(min_capacity);
+        self.tasks_by_key.shrink_to(min_capacity)
+    }
+
+    /// Look up a task in the map by its key, returning the key and abort handle.
+    fn get_by_key<'map, Q: ?Sized>(&'map self, key: &Q) -> Option<(&'map Key<K>, &'map AbortHandle)>
+    where
+        Q: Hash + Eq,
+        K: Borrow<Q>,
+    {
+        let hash = self.hash(key);
+        self.tasks_by_key
+            .raw_entry()
+            .from_hash(hash, |k| k.key.borrow() == key)
+    }
+
+    /// Look up a task in the map by its task ID, returning the key and abort handle.
+    fn get_by_id<'map>(&'map self, id: &Id) -> Option<(&'map Key<K>, &'map AbortHandle)> {
+        let hash = self.hashes_by_task.get(id)?;
+        self.tasks_by_key
+            .raw_entry()
+            .from_hash(*hash, |k| &k.id == id)
+    }
+
+    /// Remove a task from the map by ID, returning the key for that task.
+    fn remove_by_id(&mut self, id: Id) -> Option<K> {
+        // Get the hash for the given ID.
+        let hash = self.hashes_by_task.remove(&id)?;
+
+        // Remove the entry for that hash.
+        let entry = self
+            .tasks_by_key
+            .raw_entry_mut()
+            .from_hash(hash, |k| k.id == id);
+        let (Key { id: _key_id, key }, handle) = match entry {
+            RawEntryMut::Occupied(entry) => entry.remove_entry(),
+            _ => return None,
+        };
+        debug_assert_eq!(_key_id, id);
+        debug_assert_eq!(id, handle.id());
+        self.hashes_by_task.remove(&id);
+        Some(key)
+    }
+
+    /// Returns the hash for a given key.
+    #[inline]
+    fn hash<Q: ?Sized>(&self, key: &Q) -> u64
+    where
+        Q: Hash,
+    {
+        let mut hasher = self.tasks_by_key.hasher().build_hasher();
+        key.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+impl<K, V, S> JoinMap<K, V, S>
+where
+    V: 'static,
+{
+    /// Aborts all tasks on this `JoinMap`.
+    ///
+    /// This does not remove the tasks from the `JoinMap`. To wait for the tasks to complete
+    /// cancellation, you should call `join_one` in a loop until the `JoinMap` is empty.
+    pub fn abort_all(&mut self) {
+        self.tasks.abort_all()
+    }
+
+    /// Removes all tasks from this `JoinMap` without aborting them.
+    ///
+    /// The tasks removed by this call will continue to run in the background even if the `JoinMap`
+    /// is dropped. They may still be aborted by key.
+    pub fn detach_all(&mut self) {
+        self.tasks.detach_all();
+        self.tasks_by_key.clear();
+        self.hashes_by_task.clear();
+    }
+}
+
+// Hand-written `fmt::Debug` implementation in order to avoid requiring `V:
+// Debug`, since no value is ever actually stored in the map.
+impl<K: fmt::Debug, V, S> fmt::Debug for JoinMap<K, V, S> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // format the task keys and abort handles a little nicer by just
+        // printing the key and task ID pairs, without format the `Key` struct
+        // itself or the `AbortHandle`, which would just format the task's ID
+        // again.
+        struct KeySet<'a, K: fmt::Debug, S>(&'a HashMap<Key<K>, AbortHandle, S>);
+        impl<K: fmt::Debug, S> fmt::Debug for KeySet<'_, K, S> {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                f.debug_map()
+                    .entries(self.0.keys().map(|Key { key, id }| (key, id)))
+                    .finish()
+            }
+        }
+
+        f.debug_struct("JoinMap")
+            // The `tasks_by_key` map is the only one that contains information
+            // that's really worth formatting for the user, since it contains
+            // the tasks' keys and IDs. The other fields are basically
+            // implementation details.
+            .field("tasks", &KeySet(&self.tasks_by_key))
+            .finish()
+    }
+}
+
+impl<K, V> Default for JoinMap<K, V> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// === impl Key ===
+
+impl<K: Hash> Hash for Key<K> {
+    // Don't include the task ID in the hash.
+    #[inline]
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        self.key.hash(hasher);
+    }
+}
+
+// Because we override `Hash` for this type, we must also override the
+// `PartialEq` impl, so that all instances with the same hash are equal.
+impl<K: PartialEq> PartialEq for Key<K> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        self.key == other.key
+    }
+}
+
+impl<K: Eq> Eq for Key<K> {}

tokio-util/src/task/mod.rs

@@ -0,0 +1,10 @@
+//! Extra utilities for spawning tasks
+
+#[cfg(tokio_unstable)]
+mod join_map;
+mod spawn_pinned;
+pub use spawn_pinned::LocalPoolHandle;
+
+#[cfg(tokio_unstable)]
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt"))))]
+pub use join_map::JoinMap;

tokio-util/src/task/spawn_pinned.rs

@@ -0,0 +1,307 @@
+use futures_util::future::{AbortHandle, Abortable};
+use std::fmt;
+use std::fmt::{Debug, Formatter};
+use std::future::Future;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use tokio::runtime::Builder;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+use tokio::sync::oneshot;
+use tokio::task::{spawn_local, JoinHandle, LocalSet};
+
+/// A handle to a local pool, used for spawning `!Send` tasks.
+#[derive(Clone)]
+pub struct LocalPoolHandle {
+    pool: Arc<LocalPool>,
+}
+
+impl LocalPoolHandle {
+    /// Create a new pool of threads to handle `!Send` tasks. Spawn tasks onto this
+    /// pool via [`LocalPoolHandle::spawn_pinned`].
+    ///
+    /// # Panics
+    /// Panics if the pool size is less than one.
+    pub fn new(pool_size: usize) -> LocalPoolHandle {
+        assert!(pool_size > 0);
+
+        let workers = (0..pool_size)
+            .map(|_| LocalWorkerHandle::new_worker())
+            .collect();
+
+        let pool = Arc::new(LocalPool { workers });
+
+        LocalPoolHandle { pool }
+    }
+
+    /// Spawn a task onto a worker thread and pin it there so it can't be moved
+    /// off of the thread. Note that the future is not [`Send`], but the
+    /// [`FnOnce`] which creates it is.
+    ///
+    /// # Examples
+    /// ```
+    /// use std::rc::Rc;
+    /// use tokio_util::task::LocalPoolHandle;
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     // Create the local pool
+    ///     let pool = LocalPoolHandle::new(1);
+    ///
+    ///     // Spawn a !Send future onto the pool and await it
+    ///     let output = pool
+    ///         .spawn_pinned(|| {
+    ///             // Rc is !Send + !Sync
+    ///             let local_data = Rc::new("test");
+    ///
+    ///             // This future holds an Rc, so it is !Send
+    ///             async move { local_data.to_string() }
+    ///         })
+    ///         .await
+    ///         .unwrap();
+    ///
+    ///     assert_eq!(output, "test");
+    /// }
+    /// ```
+    pub fn spawn_pinned<F, Fut>(&self, create_task: F) -> JoinHandle<Fut::Output>
+    where
+        F: FnOnce() -> Fut,
+        F: Send + 'static,
+        Fut: Future + 'static,
+        Fut::Output: Send + 'static,
+    {
+        self.pool.spawn_pinned(create_task)
+    }
+}
+
+impl Debug for LocalPoolHandle {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.write_str("LocalPoolHandle")
+    }
+}
+
+struct LocalPool {
+    workers: Vec<LocalWorkerHandle>,
+}
+
+impl LocalPool {
+    /// Spawn a `?Send` future onto a worker
+    fn spawn_pinned<F, Fut>(&self, create_task: F) -> JoinHandle<Fut::Output>
+    where
+        F: FnOnce() -> Fut,
+        F: Send + 'static,
+        Fut: Future + 'static,
+        Fut::Output: Send + 'static,
+    {
+        let (sender, receiver) = oneshot::channel();
+
+        let (worker, job_guard) = self.find_and_incr_least_burdened_worker();
+        let worker_spawner = worker.spawner.clone();
+
+        // Spawn a future onto the worker's runtime so we can immediately return
+        // a join handle.
+        worker.runtime_handle.spawn(async move {
+            // Move the job guard into the task
+            let _job_guard = job_guard;
+
+            // Propagate aborts via Abortable/AbortHandle
+            let (abort_handle, abort_registration) = AbortHandle::new_pair();
+            let _abort_guard = AbortGuard(abort_handle);
+
+            // Inside the future we can't run spawn_local yet because we're not
+            // in the context of a LocalSet. We need to send create_task to the
+            // LocalSet task for spawning.
+            let spawn_task = Box::new(move || {
+                // Once we're in the LocalSet context we can call spawn_local
+                let join_handle =
+                    spawn_local(
+                        async move { Abortable::new(create_task(), abort_registration).await },
+                    );
+
+                // Send the join handle back to the spawner. If sending fails,
+                // we assume the parent task was canceled, so cancel this task
+                // as well.
+                if let Err(join_handle) = sender.send(join_handle) {
+                    join_handle.abort()
+                }
+            });
+
+            // Send the callback to the LocalSet task
+            if let Err(e) = worker_spawner.send(spawn_task) {
+                // Propagate the error as a panic in the join handle.
+                panic!("Failed to send job to worker: {}", e);
+            }
+
+            // Wait for the task's join handle
+            let join_handle = match receiver.await {
+                Ok(handle) => handle,
+                Err(e) => {
+                    // We sent the task successfully, but failed to get its
+                    // join handle... We assume something happened to the worker
+                    // and the task was not spawned. Propagate the error as a
+                    // panic in the join handle.
+                    panic!("Worker failed to send join handle: {}", e);
+                }
+            };
+
+            // Wait for the task to complete
+            let join_result = join_handle.await;
+
+            match join_result {
+                Ok(Ok(output)) => output,
+                Ok(Err(_)) => {
+                    // Pinned task was aborted. But that only happens if this
+                    // task is aborted. So this is an impossible branch.
+                    unreachable!(
+                        "Reaching this branch means this task was previously \
+                         aborted but it continued running anyways"
+                    )
+                }
+                Err(e) => {
+                    if e.is_panic() {
+                        std::panic::resume_unwind(e.into_panic());
+                    } else if e.is_cancelled() {
+                        // No one else should have the join handle, so this is
+                        // unexpected. Forward this error as a panic in the join
+                        // handle.
+                        panic!("spawn_pinned task was canceled: {}", e);
+                    } else {
+                        // Something unknown happened (not a panic or
+                        // cancellation). Forward this error as a panic in the
+                        // join handle.
+                        panic!("spawn_pinned task failed: {}", e);
+                    }
+                }
+            }
+        })
+    }
+
+    /// Find the worker with the least number of tasks, increment its task
+    /// count, and return its handle. Make sure to actually spawn a task on
+    /// the worker so the task count is kept consistent with load.
+    ///
+    /// A job count guard is also returned to ensure the task count gets
+    /// decremented when the job is done.
+    fn find_and_incr_least_burdened_worker(&self) -> (&LocalWorkerHandle, JobCountGuard) {
+        loop {
+            let (worker, task_count) = self
+                .workers
+                .iter()
+                .map(|worker| (worker, worker.task_count.load(Ordering::SeqCst)))
+                .min_by_key(|&(_, count)| count)
+                .expect("There must be more than one worker");
+
+            // Make sure the task count hasn't changed since when we choose this
+            // worker. Otherwise, restart the search.
+            if worker
+                .task_count
+                .compare_exchange(
+                    task_count,
+                    task_count + 1,
+                    Ordering::SeqCst,
+                    Ordering::Relaxed,
+                )
+                .is_ok()
+            {
+                return (worker, JobCountGuard(Arc::clone(&worker.task_count)));
+            }
+        }
+    }
+}
+
+/// Automatically decrements a worker's job count when a job finishes (when
+/// this gets dropped).
+struct JobCountGuard(Arc<AtomicUsize>);
+
+impl Drop for JobCountGuard {
+    fn drop(&mut self) {
+        // Decrement the job count
+        let previous_value = self.0.fetch_sub(1, Ordering::SeqCst);
+        debug_assert!(previous_value >= 1);
+    }
+}
+
+/// Calls abort on the handle when dropped.
+struct AbortGuard(AbortHandle);
+
+impl Drop for AbortGuard {
+    fn drop(&mut self) {
+        self.0.abort();
+    }
+}
+
+type PinnedFutureSpawner = Box<dyn FnOnce() + Send + 'static>;
+
+struct LocalWorkerHandle {
+    runtime_handle: tokio::runtime::Handle,
+    spawner: UnboundedSender<PinnedFutureSpawner>,
+    task_count: Arc<AtomicUsize>,
+}
+
+impl LocalWorkerHandle {
+    /// Create a new worker for executing pinned tasks
+    fn new_worker() -> LocalWorkerHandle {
+        let (sender, receiver) = unbounded_channel();
+        let runtime = Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("Failed to start a pinned worker thread runtime");
+        let runtime_handle = runtime.handle().clone();
+        let task_count = Arc::new(AtomicUsize::new(0));
+        let task_count_clone = Arc::clone(&task_count);
+
+        std::thread::spawn(|| Self::run(runtime, receiver, task_count_clone));
+
+        LocalWorkerHandle {
+            runtime_handle,
+            spawner: sender,
+            task_count,
+        }
+    }
+
+    fn run(
+        runtime: tokio::runtime::Runtime,
+        mut task_receiver: UnboundedReceiver<PinnedFutureSpawner>,
+        task_count: Arc<AtomicUsize>,
+    ) {
+        let local_set = LocalSet::new();
+        local_set.block_on(&runtime, async {
+            while let Some(spawn_task) = task_receiver.recv().await {
+                // Calls spawn_local(future)
+                (spawn_task)();
+            }
+        });
+
+        // If there are any tasks on the runtime associated with a LocalSet task
+        // that has already completed, but whose output has not yet been
+        // reported, let that task complete.
+        //
+        // Since the task_count is decremented when the runtime task exits,
+        // reading that counter lets us know if any such tasks completed during
+        // the call to `block_on`.
+        //
+        // Tasks on the LocalSet can't complete during this loop since they're
+        // stored on the LocalSet and we aren't accessing it.
+        let mut previous_task_count = task_count.load(Ordering::SeqCst);
+        loop {
+            // This call will also run tasks spawned on the runtime.
+            runtime.block_on(tokio::task::yield_now());
+            let new_task_count = task_count.load(Ordering::SeqCst);
+            if new_task_count == previous_task_count {
+                break;
+            } else {
+                previous_task_count = new_task_count;
+            }
+        }
+
+        // It's now no longer possible for a task on the runtime to be
+        // associated with a LocalSet task that has completed. Drop both the
+        // LocalSet and runtime to let tasks on the runtime be cancelled if and
+        // only if they are still on the LocalSet.
+        //
+        // Drop the LocalSet task first so that anyone awaiting the runtime
+        // JoinHandle will see the cancelled error after the LocalSet task
+        // destructor has completed.
+        drop(local_set);
+        drop(runtime);
+    }
+}

tokio-util/src/time/delay_queue.rs

@@ -7,10 +7,15 @@
 use crate::time::wheel::{self, Wheel};
 
 use futures_core::ready;
-use tokio::time::{error::Error, sleep_until, Duration, Instant, Sleep};
+use tokio::time::{sleep_until, Duration, Instant, Sleep};
 
+use core::ops::{Index, IndexMut};
 use slab::Slab;
 use std::cmp;
+use std::collections::HashMap;
+use std::convert::From;
+use std::fmt;
+use std::fmt::Debug;
 use std::future::Future;
 use std::marker::PhantomData;
 use std::pin::Pin;
@@ -67,7 +72,6 @@ use std::task::{self, Poll, Waker};
 /// Using `DelayQueue` to manage cache entries.
 ///
 /// ```rust,no_run
-/// use tokio::time::error::Error;
 /// use tokio_util::time::{DelayQueue, delay_queue};
 ///
 /// use futures::ready;
@@ -103,13 +107,12 @@ use std::task::{self, Poll, Waker};
 ///         }
 ///     }
 ///
-///     fn poll_purge(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
-///         while let Some(res) = ready!(self.expirations.poll_expired(cx)) {
-///             let entry = res?;
+///     fn poll_purge(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+///         while let Some(entry) = ready!(self.expirations.poll_expired(cx)) {
 ///             self.entries.remove(entry.get_ref());
 ///         }
 ///
-///         Poll::Ready(Ok(()))
+///         Poll::Ready(())
 ///     }
 /// }
 /// ```
@@ -128,7 +131,7 @@ use std::task::{self, Poll, Waker};
 #[derive(Debug)]
 pub struct DelayQueue<T> {
     /// Stores data associated with entries
-    slab: Slab<Data<T>>,
+    slab: SlabStorage<T>,
 
     /// Lookup structure tracking all delays in the queue
     wheel: Wheel<Stack<T>>,
@@ -152,6 +155,216 @@ pub struct DelayQueue<T> {
     waker: Option<Waker>,
 }
 
+#[derive(Default)]
+struct SlabStorage<T> {
+    inner: Slab<Data<T>>,
+
+    // A `compact` call requires a re-mapping of the `Key`s that were changed
+    // during the `compact` call of the `slab`. Since the keys that were given out
+    // cannot be changed retroactively we need to keep track of these re-mappings.
+    // The keys of `key_map` correspond to the old keys that were given out and
+    // the values to the `Key`s that were re-mapped by the `compact` call.
+    key_map: HashMap<Key, KeyInternal>,
+
+    // Index used to create new keys to hand out.
+    next_key_index: usize,
+
+    // Whether `compact` has been called, necessary in order to decide whether
+    // to include keys in `key_map`.
+    compact_called: bool,
+}
+
+impl<T> SlabStorage<T> {
+    pub(crate) fn with_capacity(capacity: usize) -> SlabStorage<T> {
+        SlabStorage {
+            inner: Slab::with_capacity(capacity),
+            key_map: HashMap::new(),
+            next_key_index: 0,
+            compact_called: false,
+        }
+    }
+
+    // Inserts data into the inner slab and re-maps keys if necessary
+    pub(crate) fn insert(&mut self, val: Data<T>) -> Key {
+        let mut key = KeyInternal::new(self.inner.insert(val));
+        let key_contained = self.key_map.contains_key(&key.into());
+
+        if key_contained {
+            // It's possible that a `compact` call creates capacitiy in `self.inner` in
+            // such a way that a `self.inner.insert` call creates a `key` which was
+            // previously given out during an `insert` call prior to the `compact` call.
+            // If `key` is contained in `self.key_map`, we have encountered this exact situation,
+            // We need to create a new key `key_to_give_out` and include the relation
+            // `key_to_give_out` -> `key` in `self.key_map`.
+            let key_to_give_out = self.create_new_key();
+            assert!(!self.key_map.contains_key(&key_to_give_out.into()));
+            self.key_map.insert(key_to_give_out.into(), key);
+            key = key_to_give_out;
+        } else if self.compact_called {
+            // Include an identity mapping in `self.key_map` in order to allow us to
+            // panic if a key that was handed out is removed more than once.
+            self.key_map.insert(key.into(), key);
+        }
+
+        key.into()
+    }
+
+    // Re-map the key in case compact was previously called.
+    // Note: Since we include identity mappings in key_map after compact was called,
+    // we have information about all keys that were handed out. In the case in which
+    // compact was called and we try to remove a Key that was previously removed
+    // we can detect invalid keys if no key is found in `key_map`. This is necessary
+    // in order to prevent situations in which a previously removed key
+    // corresponds to a re-mapped key internally and which would then be incorrectly
+    // removed from the slab.
+    //
+    // Example to illuminate this problem:
+    //
+    // Let's assume our `key_map` is {1 -> 2, 2 -> 1} and we call remove(1). If we
+    // were to remove 1 again, we would not find it inside `key_map` anymore.
+    // If we were to imply from this that no re-mapping was necessary, we would
+    // incorrectly remove 1 from `self.slab.inner`, which corresponds to the
+    // handed-out key 2.
+    pub(crate) fn remove(&mut self, key: &Key) -> Data<T> {
+        let remapped_key = if self.compact_called {
+            match self.key_map.remove(key) {
+                Some(key_internal) => key_internal,
+                None => panic!("invalid key"),
+            }
+        } else {
+            (*key).into()
+        };
+
+        self.inner.remove(remapped_key.index)
+    }
+
+    pub(crate) fn shrink_to_fit(&mut self) {
+        self.inner.shrink_to_fit();
+        self.key_map.shrink_to_fit();
+    }
+
+    pub(crate) fn compact(&mut self) {
+        if !self.compact_called {
+            for (key, _) in self.inner.iter() {
+                self.key_map.insert(Key::new(key), KeyInternal::new(key));
+            }
+        }
+
+        let mut remapping = HashMap::new();
+        self.inner.compact(|_, from, to| {
+            remapping.insert(from, to);
+            true
+        });
+
+        // At this point `key_map` contains a mapping for every element.
+        for internal_key in self.key_map.values_mut() {
+            if let Some(new_internal_key) = remapping.get(&internal_key.index) {
+                *internal_key = KeyInternal::new(*new_internal_key);
+            }
+        }
+
+        if self.key_map.capacity() > 2 * self.key_map.len() {
+            self.key_map.shrink_to_fit();
+        }
+
+        self.compact_called = true;
+    }
+
+    // Tries to re-map a `Key` that was given out to the user to its
+    // corresponding internal key.
+    fn remap_key(&self, key: &Key) -> Option<KeyInternal> {
+        let key_map = &self.key_map;
+        if self.compact_called {
+            key_map.get(&*key).copied()
+        } else {
+            Some((*key).into())
+        }
+    }
+
+    fn create_new_key(&mut self) -> KeyInternal {
+        while self.key_map.contains_key(&Key::new(self.next_key_index)) {
+            self.next_key_index = self.next_key_index.wrapping_add(1);
+        }
+
+        KeyInternal::new(self.next_key_index)
+    }
+
+    pub(crate) fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub(crate) fn capacity(&self) -> usize {
+        self.inner.capacity()
+    }
+
+    pub(crate) fn clear(&mut self) {
+        self.inner.clear();
+        self.key_map.clear();
+        self.compact_called = false;
+    }
+
+    pub(crate) fn reserve(&mut self, additional: usize) {
+        self.inner.reserve(additional);
+
+        if self.compact_called {
+            self.key_map.reserve(additional);
+        }
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+
+    pub(crate) fn contains(&self, key: &Key) -> bool {
+        let remapped_key = self.remap_key(key);
+
+        match remapped_key {
+            Some(internal_key) => self.inner.contains(internal_key.index),
+            None => false,
+        }
+    }
+}
+
+impl<T> fmt::Debug for SlabStorage<T>
+where
+    T: fmt::Debug,
+{
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if fmt.alternate() {
+            fmt.debug_map().entries(self.inner.iter()).finish()
+        } else {
+            fmt.debug_struct("Slab")
+                .field("len", &self.len())
+                .field("cap", &self.capacity())
+                .finish()
+        }
+    }
+}
+
+impl<T> Index<Key> for SlabStorage<T> {
+    type Output = Data<T>;
+
+    fn index(&self, key: Key) -> &Self::Output {
+        let remapped_key = self.remap_key(&key);
+
+        match remapped_key {
+            Some(internal_key) => &self.inner[internal_key.index],
+            None => panic!("Invalid index {}", key.index),
+        }
+    }
+}
+
+impl<T> IndexMut<Key> for SlabStorage<T> {
+    fn index_mut(&mut self, key: Key) -> &mut Data<T> {
+        let remapped_key = self.remap_key(&key);
+
+        match remapped_key {
+            Some(internal_key) => &mut self.inner[internal_key.index],
+            None => panic!("Invalid index {}", key.index),
+        }
+    }
+}
+
 /// An entry in `DelayQueue` that has expired and been removed.
 ///
 /// Values are returned by [`DelayQueue::poll_expired`].
@@ -176,15 +389,23 @@ pub struct Expired<T> {
 ///
 /// [`DelayQueue`]: struct@DelayQueue
 /// [`DelayQueue::insert`]: method@DelayQueue::insert
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct Key {
     index: usize,
 }
 
+// Whereas `Key` is given out to users that use `DelayQueue`, internally we use
+// `KeyInternal` as the key type in order to make the logic of mapping between keys
+// as a result of `compact` calls clearer.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct KeyInternal {
+    index: usize,
+}
+
 #[derive(Debug)]
 struct Stack<T> {
     /// Head of the stack
-    head: Option<usize>,
+    head: Option<Key>,
     _p: PhantomData<fn() -> T>,
 }
 
@@ -201,10 +422,10 @@ struct Data<T> {
     expired: bool,
 
     /// Next entry in the stack
-    next: Option<usize>,
+    next: Option<Key>,
 
     /// Previous entry in the stack
-    prev: Option<usize>,
+    prev: Option<Key>,
 }
 
 /// Maximum number of entries the queue can handle
@@ -253,7 +474,7 @@ impl<T> DelayQueue<T> {
     pub fn with_capacity(capacity: usize) -> DelayQueue<T> {
         DelayQueue {
             wheel: Wheel::new(),
-            slab: Slab::with_capacity(capacity),
+            slab: SlabStorage::with_capacity(capacity),
             expired: Stack::default(),
             delay: None,
             wheel_now: 0,
@@ -348,16 +569,13 @@ impl<T> DelayQueue<T> {
             }
         }
 
-        Key::new(key)
+        key
     }
 
     /// Attempts to pull out the next value of the delay queue, registering the
     /// current task for wakeup if the value is not yet available, and returning
     /// `None` if the queue is exhausted.
-    pub fn poll_expired(
-        &mut self,
-        cx: &mut task::Context<'_>,
-    ) -> Poll<Option<Result<Expired<T>, Error>>> {
+    pub fn poll_expired(&mut self, cx: &mut task::Context<'_>) -> Poll<Option<Expired<T>>> {
         if !self
             .waker
             .as_ref()
@@ -368,18 +586,16 @@ impl<T> DelayQueue<T> {
         }
 
         let item = ready!(self.poll_idx(cx));
-        Poll::Ready(item.map(|result| {
-            result.map(|idx| {
-                let data = self.slab.remove(idx);
-                debug_assert!(data.next.is_none());
-                debug_assert!(data.prev.is_none());
+        Poll::Ready(item.map(|key| {
+            let data = self.slab.remove(&key);
+            debug_assert!(data.next.is_none());
+            debug_assert!(data.prev.is_none());
 
-                Expired {
-                    key: Key::new(idx),
-                    data: data.inner,
-                    deadline: self.start + Duration::from_millis(data.when),
-                }
-            })
+            Expired {
+                key,
+                data: data.inner,
+                deadline: self.start + Duration::from_millis(data.when),
+            }
         }))
     }
 
@@ -437,7 +653,7 @@ impl<T> DelayQueue<T> {
         self.insert_at(value, Instant::now() + timeout)
     }
 
-    fn insert_idx(&mut self, when: u64, key: usize) {
+    fn insert_idx(&mut self, when: u64, key: Key) {
         use self::wheel::{InsertError, Stack};
 
         // Register the deadline with the timer wheel
@@ -462,10 +678,10 @@ impl<T> DelayQueue<T> {
         use crate::time::wheel::Stack;
 
         // Special case the `expired` queue
-        if self.slab[key.index].expired {
-            self.expired.remove(&key.index, &mut self.slab);
+        if self.slab[*key].expired {
+            self.expired.remove(key, &mut self.slab);
         } else {
-            self.wheel.remove(&key.index, &mut self.slab);
+            self.wheel.remove(key, &mut self.slab);
         }
     }
 
@@ -501,7 +717,7 @@ impl<T> DelayQueue<T> {
         let prev_deadline = self.next_deadline();
 
         self.remove_key(key);
-        let data = self.slab.remove(key.index);
+        let data = self.slab.remove(key);
 
         let next_deadline = self.next_deadline();
         if prev_deadline != next_deadline {
@@ -559,10 +775,10 @@ impl<T> DelayQueue<T> {
         // Normalize the deadline. Values cannot be set to expire in the past.
         let when = self.normalize_deadline(when);
 
-        self.slab[key.index].when = when;
-        self.slab[key.index].expired = false;
+        self.slab[*key].when = when;
+        self.slab[*key].expired = false;
 
-        self.insert_idx(when, key.index);
+        self.insert_idx(when, *key);
 
         let next_deadline = self.next_deadline();
         if let (Some(ref mut delay), Some(deadline)) = (&mut self.delay, next_deadline) {
@@ -571,6 +787,50 @@ impl<T> DelayQueue<T> {
         }
     }
 
+    /// Shrink the capacity of the slab, which `DelayQueue` uses internally for storage allocation.
+    /// This function is not guaranteed to, and in most cases, won't decrease the capacity of the slab
+    /// to the number of elements still contained in it, because elements cannot be moved to a different
+    /// index. To decrease the capacity to the size of the slab use [`compact`].
+    ///
+    /// This function can take O(n) time even when the capacity cannot be reduced or the allocation is
+    /// shrunk in place. Repeated calls run in O(1) though.
+    ///
+    /// [`compact`]: method@Self::compact
+    pub fn shrink_to_fit(&mut self) {
+        self.slab.shrink_to_fit();
+    }
+
+    /// Shrink the capacity of the slab, which `DelayQueue` uses internally for storage allocation,
+    /// to the number of elements that are contained in it.
+    ///
+    /// This methods runs in O(n).
+    ///
+    /// # Examples
+    ///
+    /// Basic usage
+    ///
+    /// ```rust
+    /// use tokio_util::time::DelayQueue;
+    /// use std::time::Duration;
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// let mut delay_queue = DelayQueue::with_capacity(10);
+    ///
+    /// let key1 = delay_queue.insert(5, Duration::from_secs(5));
+    /// let key2 = delay_queue.insert(10, Duration::from_secs(10));
+    /// let key3 = delay_queue.insert(15, Duration::from_secs(15));
+    ///
+    /// delay_queue.remove(&key2);
+    ///
+    /// delay_queue.compact();
+    /// assert_eq!(delay_queue.capacity(), 2);
+    /// # }
+    /// ```
+    pub fn compact(&mut self) {
+        self.slab.compact();
+    }
+
     /// Returns the next time to poll as determined by the wheel
     fn next_deadline(&mut self) -> Option<Instant> {
         self.wheel
@@ -750,13 +1010,13 @@ impl<T> DelayQueue<T> {
     /// should be returned.
     ///
     /// A slot should be returned when the associated deadline has been reached.
-    fn poll_idx(&mut self, cx: &mut task::Context<'_>) -> Poll<Option<Result<usize, Error>>> {
+    fn poll_idx(&mut self, cx: &mut task::Context<'_>) -> Poll<Option<Key>> {
         use self::wheel::Stack;
 
         let expired = self.expired.pop(&mut self.slab);
 
         if expired.is_some() {
-            return Poll::Ready(expired.map(Ok));
+            return Poll::Ready(expired);
         }
 
         loop {
@@ -776,7 +1036,7 @@ impl<T> DelayQueue<T> {
             self.delay = self.next_deadline().map(|when| Box::pin(sleep_until(when)));
 
             if let Some(idx) = wheel_idx {
-                return Poll::Ready(Some(Ok(idx)));
+                return Poll::Ready(Some(idx));
             }
 
             if self.delay.is_none() {
@@ -808,7 +1068,7 @@ impl<T> Default for DelayQueue<T> {
 impl<T> futures_core::Stream for DelayQueue<T> {
     // DelayQueue seems much more specific, where a user may care that it
     // has reached capacity, so return those errors instead of panicking.
-    type Item = Result<Expired<T>, Error>;
+    type Item = Expired<T>;
 
     fn poll_next(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Option<Self::Item>> {
         DelayQueue::poll_expired(self.get_mut(), cx)
@@ -816,9 +1076,9 @@ impl<T> futures_core::Stream for DelayQueue<T> {
 }
 
 impl<T> wheel::Stack for Stack<T> {
-    type Owned = usize;
-    type Borrowed = usize;
-    type Store = Slab<Data<T>>;
+    type Owned = Key;
+    type Borrowed = Key;
+    type Store = SlabStorage<T>;
 
     fn is_empty(&self) -> bool {
         self.head.is_none()
@@ -837,28 +1097,29 @@ impl<T> wheel::Stack for Stack<T> {
         }
 
         store[item].next = old;
-        self.head = Some(item)
+        self.head = Some(item);
     }
 
     fn pop(&mut self, store: &mut Self::Store) -> Option<Self::Owned> {
-        if let Some(idx) = self.head {
-            self.head = store[idx].next;
+        if let Some(key) = self.head {
+            self.head = store[key].next;
 
             if let Some(idx) = self.head {
                 store[idx].prev = None;
             }
 
-            store[idx].next = None;
-            debug_assert!(store[idx].prev.is_none());
+            store[key].next = None;
+            debug_assert!(store[key].prev.is_none());
 
-            Some(idx)
+            Some(key)
         } else {
             None
         }
     }
 
     fn remove(&mut self, item: &Self::Borrowed, store: &mut Self::Store) {
-        assert!(store.contains(*item));
+        let key = *item;
+        assert!(store.contains(item));
 
         // Ensure that the entry is in fact contained by the stack
         debug_assert!({
@@ -867,29 +1128,31 @@ impl<T> wheel::Stack for Stack<T> {
             let mut contains = false;
 
             while let Some(idx) = next {
+                let data = &store[idx];
+
                 if idx == *item {
                     debug_assert!(!contains);
                     contains = true;
                 }
 
-                next = store[idx].next;
+                next = data.next;
             }
 
             contains
         });
 
-        if let Some(next) = store[*item].next {
-            store[next].prev = store[*item].prev;
+        if let Some(next) = store[key].next {
+            store[next].prev = store[key].prev;
         }
 
-        if let Some(prev) = store[*item].prev {
-            store[prev].next = store[*item].next;
+        if let Some(prev) = store[key].prev {
+            store[prev].next = store[key].next;
         } else {
-            self.head = store[*item].next;
+            self.head = store[key].next;
         }
 
-        store[*item].next = None;
-        store[*item].prev = None;
+        store[key].next = None;
+        store[key].prev = None;
     }
 
     fn when(item: &Self::Borrowed, store: &Self::Store) -> u64 {
@@ -912,6 +1175,24 @@ impl Key {
     }
 }
 
+impl KeyInternal {
+    pub(crate) fn new(index: usize) -> KeyInternal {
+        KeyInternal { index }
+    }
+}
+
+impl From<Key> for KeyInternal {
+    fn from(item: Key) -> Self {
+        KeyInternal::new(item.index)
+    }
+}
+
+impl From<KeyInternal> for Key {
+    fn from(item: KeyInternal) -> Self {
+        Key::new(item.index)
+    }
+}
+
 impl<T> Expired<T> {
     /// Returns a reference to the inner value.
     pub fn get_ref(&self) -> &T {

tokio-util/src/time/wheel/mod.rs

@@ -6,6 +6,7 @@ mod stack;
 pub(crate) use self::stack::Stack;
 
 use std::borrow::Borrow;
+use std::fmt::Debug;
 use std::usize;
 
 /// Timing wheel implementation.

tokio-util/src/time/wheel/stack.rs

@@ -1,4 +1,6 @@
 use std::borrow::Borrow;
+use std::cmp::Eq;
+use std::hash::Hash;
 
 /// Abstracts the stack operations needed to track timeouts.
 pub(crate) trait Stack: Default {
@@ -6,7 +8,7 @@ pub(crate) trait Stack: Default {
     type Owned: Borrow<Self::Borrowed>;
 
     /// Borrowed item
-    type Borrowed;
+    type Borrowed: Eq + Hash;
 
     /// Item storage, this allows a slab to be used instead of just the heap
     type Store;

tokio-util/src/udp/frame.rs

@@ -81,17 +81,21 @@ where
             }
 
             // We're out of data. Try and fetch more data to decode
-            let addr = unsafe {
-                // Convert `&mut [MaybeUnit<u8>]` to `&mut [u8]` because we will be
-                // writing to it via `poll_recv_from` and therefore initializing the memory.
-                let buf = &mut *(pin.rd.chunk_mut() as *mut _ as *mut [MaybeUninit<u8>]);
+            let addr = {
+                // Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
+                // transparent wrapper around `[MaybeUninit<u8>]`.
+                let buf = unsafe { &mut *(pin.rd.chunk_mut() as *mut _ as *mut [MaybeUninit<u8>]) };
                 let mut read = ReadBuf::uninit(buf);
                 let ptr = read.filled().as_ptr();
                 let res = ready!(pin.socket.borrow().poll_recv_from(cx, &mut read));
 
                 assert_eq!(ptr, read.filled().as_ptr());
                 let addr = res?;
-                pin.rd.advance_mut(read.filled().len());
+
+                // Safety: This is guaranteed to be the number of initialized (and read) bytes due
+                // to the invariants provided by `ReadBuf::filled`.
+                unsafe { pin.rd.advance_mut(read.filled().len()) };
+
                 addr
             };
 

tokio-util/tests/codecs.rs

@@ -38,6 +38,9 @@ fn bytes_encoder() {
     codec
         .encode(Bytes::from_static(&[0; INITIAL_CAPACITY + 1]), &mut buf)
         .unwrap();
+    codec
+        .encode(BytesMut::from(&b"hello"[..]), &mut buf)
+        .unwrap();
 }
 
 #[test]

tokio-util/tests/framed.rs

@@ -12,7 +12,10 @@ use std::task::{Context, Poll};
 const INITIAL_CAPACITY: usize = 8 * 1024;
 
 /// Encode and decode u32 values.
-struct U32Codec;
+#[derive(Default)]
+struct U32Codec {
+    read_bytes: usize,
+}
 
 impl Decoder for U32Codec {
     type Item = u32;
@@ -24,6 +27,7 @@ impl Decoder for U32Codec {
         }
 
         let n = buf.split_to(4).get_u32();
+        self.read_bytes += 4;
         Ok(Some(n))
     }
 }
@@ -39,6 +43,38 @@ impl Encoder<u32> for U32Codec {
     }
 }
 
+/// Encode and decode u64 values.
+#[derive(Default)]
+struct U64Codec {
+    read_bytes: usize,
+}
+
+impl Decoder for U64Codec {
+    type Item = u64;
+    type Error = io::Error;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> io::Result<Option<u64>> {
+        if buf.len() < 8 {
+            return Ok(None);
+        }
+
+        let n = buf.split_to(8).get_u64();
+        self.read_bytes += 8;
+        Ok(Some(n))
+    }
+}
+
+impl Encoder<u64> for U64Codec {
+    type Error = io::Error;
+
+    fn encode(&mut self, item: u64, dst: &mut BytesMut) -> io::Result<()> {
+        // Reserve space
+        dst.reserve(8);
+        dst.put_u64(item);
+        Ok(())
+    }
+}
+
 /// This value should never be used
 struct DontReadIntoThis;
 
@@ -63,18 +99,39 @@ impl tokio::io::AsyncRead for DontReadIntoThis {
 
 #[tokio::test]
 async fn can_read_from_existing_buf() {
-    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec);
+    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec::default());
     parts.read_buf = BytesMut::from(&[0, 0, 0, 42][..]);
 
     let mut framed = Framed::from_parts(parts);
     let num = assert_ok!(framed.next().await.unwrap());
 
     assert_eq!(num, 42);
+    assert_eq!(framed.codec().read_bytes, 4);
+}
+
+#[tokio::test]
+async fn can_read_from_existing_buf_after_codec_changed() {
+    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec::default());
+    parts.read_buf = BytesMut::from(&[0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 84][..]);
+
+    let mut framed = Framed::from_parts(parts);
+    let num = assert_ok!(framed.next().await.unwrap());
+
+    assert_eq!(num, 42);
+    assert_eq!(framed.codec().read_bytes, 4);
+
+    let mut framed = framed.map_codec(|codec| U64Codec {
+        read_bytes: codec.read_bytes,
+    });
+    let num = assert_ok!(framed.next().await.unwrap());
+
+    assert_eq!(num, 84);
+    assert_eq!(framed.codec().read_bytes, 12);
 }
 
 #[test]
 fn external_buf_grows_to_init() {
-    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec);
+    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec::default());
     parts.read_buf = BytesMut::from(&[0, 0, 0, 42][..]);
 
     let framed = Framed::from_parts(parts);
@@ -85,7 +142,7 @@ fn external_buf_grows_to_init() {
 
 #[test]
 fn external_buf_does_not_shrink() {
-    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec);
+    let mut parts = FramedParts::new(DontReadIntoThis, U32Codec::default());
     parts.read_buf = BytesMut::from(&vec![0; INITIAL_CAPACITY * 2][..]);
 
     let framed = Framed::from_parts(parts);

tokio-util/tests/framed_read.rs

@@ -50,6 +50,22 @@ impl Decoder for U32Decoder {
     }
 }
 
+struct U64Decoder;
+
+impl Decoder for U64Decoder {
+    type Item = u64;
+    type Error = io::Error;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> io::Result<Option<u64>> {
+        if buf.len() < 8 {
+            return Ok(None);
+        }
+
+        let n = buf.split_to(8).get_u64();
+        Ok(Some(n))
+    }
+}
+
 #[test]
 fn read_multi_frame_in_packet() {
     let mut task = task::spawn(());
@@ -84,6 +100,24 @@ fn read_multi_frame_across_packets() {
     });
 }
 
+#[test]
+fn read_multi_frame_in_packet_after_codec_changed() {
+    let mut task = task::spawn(());
+    let mock = mock! {
+        Ok(b"\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08".to_vec()),
+    };
+    let mut framed = FramedRead::new(mock, U32Decoder);
+
+    task.enter(|cx, _| {
+        assert_read!(pin!(framed).poll_next(cx), 0x04);
+
+        let mut framed = framed.map_decoder(|_| U64Decoder);
+        assert_read!(pin!(framed).poll_next(cx), 0x08);
+
+        assert!(assert_ready!(pin!(framed).poll_next(cx)).is_none());
+    });
+}
+
 #[test]
 fn read_not_ready() {
     let mut task = task::spawn(());

tokio-util/tests/framed_write.rs

@@ -39,6 +39,19 @@ impl Encoder<u32> for U32Encoder {
     }
 }
 
+struct U64Encoder;
+
+impl Encoder<u64> for U64Encoder {
+    type Error = io::Error;
+
+    fn encode(&mut self, item: u64, dst: &mut BytesMut) -> io::Result<()> {
+        // Reserve space
+        dst.reserve(8);
+        dst.put_u64(item);
+        Ok(())
+    }
+}
+
 #[test]
 fn write_multi_frame_in_packet() {
     let mut task = task::spawn(());
@@ -65,6 +78,32 @@ fn write_multi_frame_in_packet() {
     });
 }
 
+#[test]
+fn write_multi_frame_after_codec_changed() {
+    let mut task = task::spawn(());
+    let mock = mock! {
+        Ok(b"\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x08".to_vec()),
+    };
+    let mut framed = FramedWrite::new(mock, U32Encoder);
+
+    task.enter(|cx, _| {
+        assert!(assert_ready!(pin!(framed).poll_ready(cx)).is_ok());
+        assert!(pin!(framed).start_send(0x04).is_ok());
+
+        let mut framed = framed.map_encoder(|_| U64Encoder);
+        assert!(assert_ready!(pin!(framed).poll_ready(cx)).is_ok());
+        assert!(pin!(framed).start_send(0x08).is_ok());
+
+        // Nothing written yet
+        assert_eq!(1, framed.get_ref().calls.len());
+
+        // Flush the writes
+        assert!(assert_ready!(pin!(framed).poll_flush(cx)).is_ok());
+
+        assert_eq!(0, framed.get_ref().calls.len());
+    });
+}
+
 #[test]
 fn write_hits_backpressure() {
     const ITER: usize = 2 * 1024;

tokio-util/tests/mpsc.rs

@@ -5,53 +5,62 @@ use tokio_test::{assert_pending, assert_ready, assert_ready_err, assert_ready_ok
 use tokio_util::sync::PollSender;
 
 #[tokio::test]
-async fn test_simple() {
+async fn simple() {
     let (send, mut recv) = channel(3);
     let mut send = PollSender::new(send);
 
     for i in 1..=3i32 {
-        send.start_send(i).unwrap();
-        assert_ready_ok!(spawn(poll_fn(|cx| send.poll_send_done(cx))).poll());
+        let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+        assert_ready_ok!(reserve.poll());
+        send.send_item(i).unwrap();
     }
 
-    send.start_send(4).unwrap();
-    let mut fourth_send = spawn(poll_fn(|cx| send.poll_send_done(cx)));
-    assert_pending!(fourth_send.poll());
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+
     assert_eq!(recv.recv().await.unwrap(), 1);
-    assert!(fourth_send.is_woken());
-    assert_ready_ok!(fourth_send.poll());
+    assert!(reserve.is_woken());
+    assert_ready_ok!(reserve.poll());
 
     drop(recv);
 
-    // Here, start_send is not guaranteed to fail, but if it doesn't the first
-    // call to poll_send_done should.
-    if send.start_send(5).is_ok() {
-        assert_ready_err!(spawn(poll_fn(|cx| send.poll_send_done(cx))).poll());
-    }
+    send.send_item(42).unwrap();
 }
 
 #[tokio::test]
-async fn test_abort() {
+async fn repeated_poll_reserve() {
+    let (send, mut recv) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    assert_ready_ok!(reserve.poll());
+    send.send_item(1).unwrap();
+
+    assert_eq!(recv.recv().await.unwrap(), 1);
+}
+
+#[tokio::test]
+async fn abort_send() {
     let (send, mut recv) = channel(3);
     let mut send = PollSender::new(send);
-    let send2 = send.clone_inner().unwrap();
+    let send2 = send.get_ref().cloned().unwrap();
 
     for i in 1..=3i32 {
-        send.start_send(i).unwrap();
-        assert_ready_ok!(spawn(poll_fn(|cx| send.poll_send_done(cx))).poll());
+        let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+        assert_ready_ok!(reserve.poll());
+        send.send_item(i).unwrap();
     }
 
-    send.start_send(4).unwrap();
-    {
-        let mut fourth_send = spawn(poll_fn(|cx| send.poll_send_done(cx)));
-        assert_pending!(fourth_send.poll());
-        assert_eq!(recv.recv().await.unwrap(), 1);
-        assert!(fourth_send.is_woken());
-    }
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+    assert_eq!(recv.recv().await.unwrap(), 1);
+    assert!(reserve.is_woken());
+    assert_ready_ok!(reserve.poll());
 
     let mut send2_send = spawn(send2.send(5));
     assert_pending!(send2_send.poll());
-    send.abort_send();
+    assert!(send.abort_send());
     assert!(send2_send.is_woken());
     assert_ready_ok!(send2_send.poll());
 
@@ -68,7 +77,7 @@ async fn close_sender_last() {
     let mut recv_task = spawn(recv.recv());
     assert_pending!(recv_task.poll());
 
-    send.close_this_sender();
+    send.close();
 
     assert!(recv_task.is_woken());
     assert!(assert_ready!(recv_task.poll()).is_none());
@@ -77,13 +86,13 @@ async fn close_sender_last() {
 #[tokio::test]
 async fn close_sender_not_last() {
     let (send, mut recv) = channel::<i32>(3);
-    let send2 = send.clone();
     let mut send = PollSender::new(send);
+    let send2 = send.get_ref().cloned().unwrap();
 
     let mut recv_task = spawn(recv.recv());
     assert_pending!(recv_task.poll());
 
-    send.close_this_sender();
+    send.close();
 
     assert!(!recv_task.is_woken());
     assert_pending!(recv_task.poll());
@@ -93,3 +102,138 @@ async fn close_sender_not_last() {
     assert!(recv_task.is_woken());
     assert!(assert_ready!(recv_task.poll()).is_none());
 }
+
+#[tokio::test]
+async fn close_sender_before_reserve() {
+    let (send, mut recv) = channel::<i32>(3);
+    let mut send = PollSender::new(send);
+
+    let mut recv_task = spawn(recv.recv());
+    assert_pending!(recv_task.poll());
+
+    send.close();
+
+    assert!(recv_task.is_woken());
+    assert!(assert_ready!(recv_task.poll()).is_none());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_err!(reserve.poll());
+}
+
+#[tokio::test]
+async fn close_sender_after_pending_reserve() {
+    let (send, mut recv) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut recv_task = spawn(recv.recv());
+    assert_pending!(recv_task.poll());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    send.send_item(1).unwrap();
+
+    assert!(recv_task.is_woken());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+    drop(reserve);
+
+    send.close();
+
+    assert!(send.is_closed());
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_err!(reserve.poll());
+}
+
+#[tokio::test]
+async fn close_sender_after_successful_reserve() {
+    let (send, mut recv) = channel::<i32>(3);
+    let mut send = PollSender::new(send);
+
+    let mut recv_task = spawn(recv.recv());
+    assert_pending!(recv_task.poll());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    drop(reserve);
+
+    send.close();
+    assert!(send.is_closed());
+    assert!(!recv_task.is_woken());
+    assert_pending!(recv_task.poll());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+}
+
+#[tokio::test]
+async fn abort_send_after_pending_reserve() {
+    let (send, mut recv) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut recv_task = spawn(recv.recv());
+    assert_pending!(recv_task.poll());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    send.send_item(1).unwrap();
+
+    assert_eq!(send.get_ref().unwrap().capacity(), 0);
+    assert!(!send.abort_send());
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+
+    assert!(send.abort_send());
+    assert_eq!(send.get_ref().unwrap().capacity(), 0);
+}
+
+#[tokio::test]
+async fn abort_send_after_successful_reserve() {
+    let (send, mut recv) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut recv_task = spawn(recv.recv());
+    assert_pending!(recv_task.poll());
+
+    assert_eq!(send.get_ref().unwrap().capacity(), 1);
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    assert_eq!(send.get_ref().unwrap().capacity(), 0);
+
+    assert!(send.abort_send());
+    assert_eq!(send.get_ref().unwrap().capacity(), 1);
+}
+
+#[tokio::test]
+async fn closed_when_receiver_drops() {
+    let (send, _) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_err!(reserve.poll());
+}
+
+#[should_panic]
+#[test]
+fn start_send_panics_when_idle() {
+    let (send, _) = channel::<i32>(3);
+    let mut send = PollSender::new(send);
+
+    send.send_item(1).unwrap();
+}
+
+#[should_panic]
+#[test]
+fn start_send_panics_when_acquiring() {
+    let (send, _) = channel::<i32>(1);
+    let mut send = PollSender::new(send);
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_ready_ok!(reserve.poll());
+    send.send_item(1).unwrap();
+
+    let mut reserve = spawn(poll_fn(|cx| send.poll_reserve(cx)));
+    assert_pending!(reserve.poll());
+    send.send_item(2).unwrap();
+}

tokio-util/tests/reusable_box.rs

@@ -1,10 +1,23 @@
 use futures::future::FutureExt;
 use std::alloc::Layout;
 use std::future::Future;
+use std::marker::PhantomPinned;
 use std::pin::Pin;
+use std::rc::Rc;
 use std::task::{Context, Poll};
 use tokio_util::sync::ReusableBoxFuture;
 
+#[test]
+// Clippy false positive; it's useful to be able to test the trait impls for any lifetime
+#[allow(clippy::extra_unused_lifetimes)]
+fn traits<'a>() {
+    fn assert_traits<T: Send + Sync + Unpin>() {}
+    // Use a type that is !Unpin
+    assert_traits::<ReusableBoxFuture<'a, PhantomPinned>>();
+    // Use a type that is !Send + !Sync
+    assert_traits::<ReusableBoxFuture<'a, Rc<()>>>();
+}
+
 #[test]
 fn test_different_futures() {
     let fut = async move { 10 };

tokio-util/tests/spawn_pinned.rs

@@ -0,0 +1,193 @@
+#![warn(rust_2018_idioms)]
+
+use std::rc::Rc;
+use std::sync::Arc;
+use tokio_util::task;
+
+/// Simple test of running a !Send future via spawn_pinned
+#[tokio::test]
+async fn can_spawn_not_send_future() {
+    let pool = task::LocalPoolHandle::new(1);
+
+    let output = pool
+        .spawn_pinned(|| {
+            // Rc is !Send + !Sync
+            let local_data = Rc::new("test");
+
+            // This future holds an Rc, so it is !Send
+            async move { local_data.to_string() }
+        })
+        .await
+        .unwrap();
+
+    assert_eq!(output, "test");
+}
+
+/// Dropping the join handle still lets the task execute
+#[test]
+fn can_drop_future_and_still_get_output() {
+    let pool = task::LocalPoolHandle::new(1);
+    let (sender, receiver) = std::sync::mpsc::channel();
+
+    let _ = pool.spawn_pinned(move || {
+        // Rc is !Send + !Sync
+        let local_data = Rc::new("test");
+
+        // This future holds an Rc, so it is !Send
+        async move {
+            let _ = sender.send(local_data.to_string());
+        }
+    });
+
+    assert_eq!(receiver.recv(), Ok("test".to_string()));
+}
+
+#[test]
+#[should_panic(expected = "assertion failed: pool_size > 0")]
+fn cannot_create_zero_sized_pool() {
+    let _pool = task::LocalPoolHandle::new(0);
+}
+
+/// We should be able to spawn multiple futures onto the pool at the same time.
+#[tokio::test]
+async fn can_spawn_multiple_futures() {
+    let pool = task::LocalPoolHandle::new(2);
+
+    let join_handle1 = pool.spawn_pinned(|| {
+        let local_data = Rc::new("test1");
+        async move { local_data.to_string() }
+    });
+    let join_handle2 = pool.spawn_pinned(|| {
+        let local_data = Rc::new("test2");
+        async move { local_data.to_string() }
+    });
+
+    assert_eq!(join_handle1.await.unwrap(), "test1");
+    assert_eq!(join_handle2.await.unwrap(), "test2");
+}
+
+/// A panic in the spawned task causes the join handle to return an error.
+/// But, you can continue to spawn tasks.
+#[tokio::test]
+async fn task_panic_propagates() {
+    let pool = task::LocalPoolHandle::new(1);
+
+    let join_handle = pool.spawn_pinned(|| async {
+        panic!("Test panic");
+    });
+
+    let result = join_handle.await;
+    assert!(result.is_err());
+    let error = result.unwrap_err();
+    assert!(error.is_panic());
+    let panic_str: &str = *error.into_panic().downcast().unwrap();
+    assert_eq!(panic_str, "Test panic");
+
+    // Trying again with a "safe" task still works
+    let join_handle = pool.spawn_pinned(|| async { "test" });
+    let result = join_handle.await;
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), "test");
+}
+
+/// A panic during task creation causes the join handle to return an error.
+/// But, you can continue to spawn tasks.
+#[tokio::test]
+async fn callback_panic_does_not_kill_worker() {
+    let pool = task::LocalPoolHandle::new(1);
+
+    let join_handle = pool.spawn_pinned(|| {
+        panic!("Test panic");
+        #[allow(unreachable_code)]
+        async {}
+    });
+
+    let result = join_handle.await;
+    assert!(result.is_err());
+    let error = result.unwrap_err();
+    assert!(error.is_panic());
+    let panic_str: &str = *error.into_panic().downcast().unwrap();
+    assert_eq!(panic_str, "Test panic");
+
+    // Trying again with a "safe" callback works
+    let join_handle = pool.spawn_pinned(|| async { "test" });
+    let result = join_handle.await;
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), "test");
+}
+
+/// Canceling the task via the returned join handle cancels the spawned task
+/// (which has a different, internal join handle).
+#[tokio::test]
+async fn task_cancellation_propagates() {
+    let pool = task::LocalPoolHandle::new(1);
+    let notify_dropped = Arc::new(());
+    let weak_notify_dropped = Arc::downgrade(&notify_dropped);
+
+    let (start_sender, start_receiver) = tokio::sync::oneshot::channel();
+    let (drop_sender, drop_receiver) = tokio::sync::oneshot::channel::<()>();
+    let join_handle = pool.spawn_pinned(|| async move {
+        let _drop_sender = drop_sender;
+        // Move the Arc into the task
+        let _notify_dropped = notify_dropped;
+        let _ = start_sender.send(());
+
+        // Keep the task running until it gets aborted
+        futures::future::pending::<()>().await;
+    });
+
+    // Wait for the task to start
+    let _ = start_receiver.await;
+
+    join_handle.abort();
+
+    // Wait for the inner task to abort, dropping the sender.
+    // The top level join handle aborts quicker than the inner task (the abort
+    // needs to propagate and get processed on the worker thread), so we can't
+    // just await the top level join handle.
+    let _ = drop_receiver.await;
+
+    // Check that the Arc has been dropped. This verifies that the inner task
+    // was canceled as well.
+    assert!(weak_notify_dropped.upgrade().is_none());
+}
+
+/// Tasks should be given to the least burdened worker. When spawning two tasks
+/// on a pool with two empty workers the tasks should be spawned on separate
+/// workers.
+#[tokio::test]
+async fn tasks_are_balanced() {
+    let pool = task::LocalPoolHandle::new(2);
+
+    // Spawn a task so one thread has a task count of 1
+    let (start_sender1, start_receiver1) = tokio::sync::oneshot::channel();
+    let (end_sender1, end_receiver1) = tokio::sync::oneshot::channel();
+    let join_handle1 = pool.spawn_pinned(|| async move {
+        let _ = start_sender1.send(());
+        let _ = end_receiver1.await;
+        std::thread::current().id()
+    });
+
+    // Wait for the first task to start up
+    let _ = start_receiver1.await;
+
+    // This task should be spawned on the other thread
+    let (start_sender2, start_receiver2) = tokio::sync::oneshot::channel();
+    let join_handle2 = pool.spawn_pinned(|| async move {
+        let _ = start_sender2.send(());
+        std::thread::current().id()
+    });
+
+    // Wait for the second task to start up
+    let _ = start_receiver2.await;
+
+    // Allow the first task to end
+    let _ = end_sender1.send(());
+
+    let thread_id1 = join_handle1.await.unwrap();
+    let thread_id2 = join_handle2.await.unwrap();
+
+    // Since the first task was active when the second task spawned, they should
+    // be on separate workers/threads.
+    assert_ne!(thread_id1, thread_id2);
+}

tokio-util/tests/sync_cancellation_token.rs

@@ -1,7 +1,7 @@
 #![warn(rust_2018_idioms)]
 
 use tokio::pin;
-use tokio_util::sync::CancellationToken;
+use tokio_util::sync::{CancellationToken, WaitForCancellationFuture};
 
 use core::future::Future;
 use core::task::{Context, Poll};
@@ -77,6 +77,46 @@ fn cancel_child_token_through_parent() {
     );
 }
 
+#[test]
+fn cancel_grandchild_token_through_parent_if_child_was_dropped() {
+    let (waker, wake_counter) = new_count_waker();
+    let token = CancellationToken::new();
+
+    let intermediate_token = token.child_token();
+    let child_token = intermediate_token.child_token();
+    drop(intermediate_token);
+    assert!(!child_token.is_cancelled());
+
+    let child_fut = child_token.cancelled();
+    pin!(child_fut);
+    let parent_fut = token.cancelled();
+    pin!(parent_fut);
+
+    assert_eq!(
+        Poll::Pending,
+        child_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    token.cancel();
+    assert_eq!(wake_counter, 2);
+    assert!(token.is_cancelled());
+    assert!(child_token.is_cancelled());
+
+    assert_eq!(
+        Poll::Ready(()),
+        child_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+}
+
 #[test]
 fn cancel_child_token_without_parent() {
     let (waker, wake_counter) = new_count_waker();
@@ -206,6 +246,134 @@ fn drop_multiple_child_tokens() {
     }
 }
 
+#[test]
+fn cancel_only_all_descendants() {
+    // ARRANGE
+    let (waker, wake_counter) = new_count_waker();
+
+    let parent_token = CancellationToken::new();
+    let token = parent_token.child_token();
+    let sibling_token = parent_token.child_token();
+    let child1_token = token.child_token();
+    let child2_token = token.child_token();
+    let grandchild_token = child1_token.child_token();
+    let grandchild2_token = child1_token.child_token();
+    let grandgrandchild_token = grandchild_token.child_token();
+
+    assert!(!parent_token.is_cancelled());
+    assert!(!token.is_cancelled());
+    assert!(!sibling_token.is_cancelled());
+    assert!(!child1_token.is_cancelled());
+    assert!(!child2_token.is_cancelled());
+    assert!(!grandchild_token.is_cancelled());
+    assert!(!grandchild2_token.is_cancelled());
+    assert!(!grandgrandchild_token.is_cancelled());
+
+    let parent_fut = parent_token.cancelled();
+    let fut = token.cancelled();
+    let sibling_fut = sibling_token.cancelled();
+    let child1_fut = child1_token.cancelled();
+    let child2_fut = child2_token.cancelled();
+    let grandchild_fut = grandchild_token.cancelled();
+    let grandchild2_fut = grandchild2_token.cancelled();
+    let grandgrandchild_fut = grandgrandchild_token.cancelled();
+
+    pin!(parent_fut);
+    pin!(fut);
+    pin!(sibling_fut);
+    pin!(child1_fut);
+    pin!(child2_fut);
+    pin!(grandchild_fut);
+    pin!(grandchild2_fut);
+    pin!(grandgrandchild_fut);
+
+    assert_eq!(
+        Poll::Pending,
+        parent_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        sibling_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        child1_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        child2_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        grandchild2_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Pending,
+        grandgrandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 0);
+
+    // ACT
+    token.cancel();
+
+    // ASSERT
+    assert_eq!(wake_counter, 6);
+    assert!(!parent_token.is_cancelled());
+    assert!(token.is_cancelled());
+    assert!(!sibling_token.is_cancelled());
+    assert!(child1_token.is_cancelled());
+    assert!(child2_token.is_cancelled());
+    assert!(grandchild_token.is_cancelled());
+    assert!(grandchild2_token.is_cancelled());
+    assert!(grandgrandchild_token.is_cancelled());
+
+    assert_eq!(
+        Poll::Ready(()),
+        fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        child1_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        child2_fut.as_mut().poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        grandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        grandchild2_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(
+        Poll::Ready(()),
+        grandgrandchild_fut
+            .as_mut()
+            .poll(&mut Context::from_waker(&waker))
+    );
+    assert_eq!(wake_counter, 6);
+}
+
 #[test]
 fn drop_parent_before_child_tokens() {
     let token = CancellationToken::new();
@@ -218,3 +386,15 @@ fn drop_parent_before_child_tokens() {
     drop(child1);
     drop(child2);
 }
+
+#[test]
+fn derives_send_sync() {
+    fn assert_send<T: Send>() {}
+    fn assert_sync<T: Sync>() {}
+
+    assert_send::<CancellationToken>();
+    assert_sync::<CancellationToken>();
+
+    assert_send::<WaitForCancellationFuture<'static>>();
+    assert_sync::<WaitForCancellationFuture<'static>>();
+}

tokio-util/tests/task_join_map.rs

@@ -0,0 +1,275 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "rt", tokio_unstable))]
+
+use tokio::sync::oneshot;
+use tokio::time::Duration;
+use tokio_util::task::JoinMap;
+
+use futures::future::FutureExt;
+
+fn rt() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_current_thread()
+        .build()
+        .unwrap()
+}
+
+#[tokio::test(start_paused = true)]
+async fn test_with_sleep() {
+    let mut map = JoinMap::new();
+
+    for i in 0..10 {
+        map.spawn(i, async move { i });
+        assert_eq!(map.len(), 1 + i);
+    }
+    map.detach_all();
+    assert_eq!(map.len(), 0);
+
+    assert!(matches!(map.join_one().await, None));
+
+    for i in 0..10 {
+        map.spawn(i, async move {
+            tokio::time::sleep(Duration::from_secs(i as u64)).await;
+            i
+        });
+        assert_eq!(map.len(), 1 + i);
+    }
+
+    let mut seen = [false; 10];
+    while let Some((k, res)) = map.join_one().await {
+        seen[k] = true;
+        assert_eq!(res.expect("task should have completed successfully"), k);
+    }
+
+    for was_seen in &seen {
+        assert!(was_seen);
+    }
+    assert!(matches!(map.join_one().await, None));
+
+    // Do it again.
+    for i in 0..10 {
+        map.spawn(i, async move {
+            tokio::time::sleep(Duration::from_secs(i as u64)).await;
+            i
+        });
+    }
+
+    let mut seen = [false; 10];
+    while let Some((k, res)) = map.join_one().await {
+        seen[k] = true;
+        assert_eq!(res.expect("task should have completed successfully"), k);
+    }
+
+    for was_seen in &seen {
+        assert!(was_seen);
+    }
+    assert!(matches!(map.join_one().await, None));
+}
+
+#[tokio::test]
+async fn test_abort_on_drop() {
+    let mut map = JoinMap::new();
+
+    let mut recvs = Vec::new();
+
+    for i in 0..16 {
+        let (send, recv) = oneshot::channel::<()>();
+        recvs.push(recv);
+
+        map.spawn(i, async {
+            // This task will never complete on its own.
+            futures::future::pending::<()>().await;
+            drop(send);
+        });
+    }
+
+    drop(map);
+
+    for recv in recvs {
+        // The task is aborted soon and we will receive an error.
+        assert!(recv.await.is_err());
+    }
+}
+
+#[tokio::test]
+async fn alternating() {
+    let mut map = JoinMap::new();
+
+    assert_eq!(map.len(), 0);
+    map.spawn(1, async {});
+    assert_eq!(map.len(), 1);
+    map.spawn(2, async {});
+    assert_eq!(map.len(), 2);
+
+    for i in 0..16 {
+        let (_, res) = map.join_one().await.unwrap();
+        assert!(res.is_ok());
+        assert_eq!(map.len(), 1);
+        map.spawn(i, async {});
+        assert_eq!(map.len(), 2);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn abort_by_key() {
+    let mut map = JoinMap::new();
+    let mut num_canceled = 0;
+    let mut num_completed = 0;
+    for i in 0..16 {
+        map.spawn(i, async move {
+            tokio::time::sleep(Duration::from_secs(i as u64)).await;
+        });
+    }
+
+    for i in 0..16 {
+        if i % 2 != 0 {
+            // abort odd-numbered tasks.
+            map.abort(&i);
+        }
+    }
+
+    while let Some((key, res)) = map.join_one().await {
+        match res {
+            Ok(()) => {
+                num_completed += 1;
+                assert_eq!(key % 2, 0);
+                assert!(!map.contains_key(&key));
+            }
+            Err(e) => {
+                num_canceled += 1;
+                assert!(e.is_cancelled());
+                assert_ne!(key % 2, 0);
+                assert!(!map.contains_key(&key));
+            }
+        }
+    }
+
+    assert_eq!(num_canceled, 8);
+    assert_eq!(num_completed, 8);
+}
+
+#[tokio::test(start_paused = true)]
+async fn abort_by_predicate() {
+    let mut map = JoinMap::new();
+    let mut num_canceled = 0;
+    let mut num_completed = 0;
+    for i in 0..16 {
+        map.spawn(i, async move {
+            tokio::time::sleep(Duration::from_secs(i as u64)).await;
+        });
+    }
+
+    // abort odd-numbered tasks.
+    map.abort_matching(|key| key % 2 != 0);
+
+    while let Some((key, res)) = map.join_one().await {
+        match res {
+            Ok(()) => {
+                num_completed += 1;
+                assert_eq!(key % 2, 0);
+                assert!(!map.contains_key(&key));
+            }
+            Err(e) => {
+                num_canceled += 1;
+                assert!(e.is_cancelled());
+                assert_ne!(key % 2, 0);
+                assert!(!map.contains_key(&key));
+            }
+        }
+    }
+
+    assert_eq!(num_canceled, 8);
+    assert_eq!(num_completed, 8);
+}
+
+#[test]
+fn runtime_gone() {
+    let mut map = JoinMap::new();
+    {
+        let rt = rt();
+        map.spawn_on("key", async { 1 }, rt.handle());
+        drop(rt);
+    }
+
+    let (key, res) = rt().block_on(map.join_one()).unwrap();
+    assert_eq!(key, "key");
+    assert!(res.unwrap_err().is_cancelled());
+}
+
+// This ensures that `join_one` works correctly when the coop budget is
+// exhausted.
+#[tokio::test(flavor = "current_thread")]
+async fn join_map_coop() {
+    // Large enough to trigger coop.
+    const TASK_NUM: u32 = 1000;
+
+    static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0);
+
+    let mut map = JoinMap::new();
+
+    for i in 0..TASK_NUM {
+        map.spawn(i, async move {
+            SEM.add_permits(1);
+            i
+        });
+    }
+
+    // Wait for all tasks to complete.
+    //
+    // Since this is a `current_thread` runtime, there's no race condition
+    // between the last permit being added and the task completing.
+    let _ = SEM.acquire_many(TASK_NUM).await.unwrap();
+
+    let mut count = 0;
+    let mut coop_count = 0;
+    loop {
+        match map.join_one().now_or_never() {
+            Some(Some((key, Ok(i)))) => assert_eq!(key, i),
+            Some(Some((key, Err(err)))) => panic!("failed[{}]: {}", key, err),
+            None => {
+                coop_count += 1;
+                tokio::task::yield_now().await;
+                continue;
+            }
+            Some(None) => break,
+        }
+
+        count += 1;
+    }
+    assert!(coop_count >= 1);
+    assert_eq!(count, TASK_NUM);
+}
+
+#[tokio::test(start_paused = true)]
+async fn abort_all() {
+    let mut map: JoinMap<usize, ()> = JoinMap::new();
+
+    for i in 0..5 {
+        map.spawn(i, futures::future::pending());
+    }
+    for i in 5..10 {
+        map.spawn(i, async {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        });
+    }
+
+    // The join map will now have 5 pending tasks and 5 ready tasks.
+    tokio::time::sleep(Duration::from_secs(2)).await;
+
+    map.abort_all();
+    assert_eq!(map.len(), 10);
+
+    let mut count = 0;
+    let mut seen = [false; 10];
+    while let Some((k, res)) = map.join_one().await {
+        seen[k] = true;
+        if let Err(err) = res {
+            assert!(err.is_cancelled());
+        }
+        count += 1;
+    }
+    assert_eq!(count, 10);
+    assert_eq!(map.len(), 0);
+    for was_seen in &seen {
+        assert!(was_seen);
+    }
+}

tokio-util/tests/time_delay_queue.rs

@@ -3,7 +3,7 @@
 #![cfg(feature = "full")]
 
 use tokio::time::{self, sleep, sleep_until, Duration, Instant};
-use tokio_test::{assert_ok, assert_pending, assert_ready, task};
+use tokio_test::{assert_pending, assert_ready, task};
 use tokio_util::time::DelayQueue;
 
 macro_rules! poll {
@@ -12,12 +12,12 @@ macro_rules! poll {
     };
 }
 
-macro_rules! assert_ready_ok {
+macro_rules! assert_ready_some {
     ($e:expr) => {{
-        assert_ok!(match assert_ready!($e) {
+        match assert_ready!($e) {
             Some(v) => v,
             None => panic!("None"),
-        })
+        }
     }};
 }
 
@@ -31,7 +31,7 @@ async fn single_immediate_delay() {
     // Advance time by 1ms to handle thee rounding
     sleep(ms(1)).await;
 
-    assert_ready_ok!(poll!(queue));
+    assert_ready_some!(poll!(queue));
 
     let entry = assert_ready!(poll!(queue));
     assert!(entry.is_none())
@@ -52,7 +52,7 @@ async fn multi_immediate_delays() {
     let mut res = vec![];
 
     while res.len() < 3 {
-        let entry = assert_ready_ok!(poll!(queue));
+        let entry = assert_ready_some!(poll!(queue));
         res.push(entry.into_inner());
     }
 
@@ -83,7 +83,7 @@ async fn single_short_delay() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue));
+    let entry = assert_ready_some!(poll!(queue));
     assert_eq!(*entry.get_ref(), "foo");
 
     let entry = assert_ready!(poll!(queue));
@@ -109,6 +109,7 @@ async fn multi_delay_at_start() {
 
     let start = Instant::now();
     for elapsed in 0..1200 {
+        println!("elapsed: {:?}", elapsed);
         let elapsed = elapsed + 1;
         tokio::time::sleep_until(start + ms(elapsed)).await;
 
@@ -128,10 +129,12 @@ async fn multi_delay_at_start() {
             assert_pending!(poll!(queue));
         }
     }
+    println!("finished multi_delay_start");
 }
 
 #[tokio::test]
 async fn insert_in_past_fires_immediately() {
+    println!("running insert_in_past_fires_immediately");
     time::pause();
 
     let mut queue = task::spawn(DelayQueue::new());
@@ -142,6 +145,7 @@ async fn insert_in_past_fires_immediately() {
     queue.insert_at("foo", now);
 
     assert_ready!(poll!(queue));
+    println!("finished insert_in_past_fires_immediately");
 }
 
 #[tokio::test]
@@ -189,7 +193,7 @@ async fn reset_entry() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue));
+    let entry = assert_ready_some!(poll!(queue));
     assert_eq!(*entry.get_ref(), "foo");
 
     let entry = assert_ready!(poll!(queue));
@@ -267,7 +271,7 @@ async fn repeatedly_reset_entry_inserted_as_expired() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "foo");
 
     let entry = assert_ready!(poll!(queue));
@@ -307,7 +311,7 @@ async fn remove_at_timer_wheel_threshold() {
 
     sleep(ms(80)).await;
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
 
     match entry {
         "foo" => {
@@ -344,7 +348,7 @@ async fn expires_before_last_insert() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "bar");
 }
 
@@ -371,14 +375,14 @@ async fn multi_reset() {
 
     sleep(ms(50)).await;
 
-    let entry = assert_ready_ok!(poll!(queue));
+    let entry = assert_ready_some!(poll!(queue));
     assert_eq!(*entry.get_ref(), "two");
 
     assert_pending!(poll!(queue));
 
     sleep(ms(50)).await;
 
-    let entry = assert_ready_ok!(poll!(queue));
+    let entry = assert_ready_some!(poll!(queue));
     assert_eq!(*entry.get_ref(), "one");
 
     let entry = assert_ready!(poll!(queue));
@@ -404,7 +408,7 @@ async fn expire_first_key_when_reset_to_expire_earlier() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "one");
 }
 
@@ -427,7 +431,7 @@ async fn expire_second_key_when_reset_to_expire_earlier() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "two");
 }
 
@@ -449,7 +453,7 @@ async fn reset_first_expiring_item_to_expire_later() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "two");
 }
 
@@ -475,7 +479,7 @@ async fn insert_before_first_after_poll() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "two");
 }
 
@@ -500,7 +504,7 @@ async fn insert_after_ready_poll() {
     let mut res = vec![];
 
     while res.len() < 3 {
-        let entry = assert_ready_ok!(poll!(queue));
+        let entry = assert_ready_some!(poll!(queue));
         res.push(entry.into_inner());
         queue.insert_at("foo", now + ms(500));
     }
@@ -545,7 +549,7 @@ async fn reset_later_after_slot_starts() {
     sleep(ms(1)).await;
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "foo");
 }
 
@@ -564,7 +568,7 @@ async fn reset_inserted_expired() {
 
     sleep(ms(200)).await;
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "foo");
 
     assert_eq!(queue.len(), 0);
@@ -603,7 +607,7 @@ async fn reset_earlier_after_slot_starts() {
     sleep(ms(1)).await;
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "foo");
 }
 
@@ -626,7 +630,7 @@ async fn insert_in_past_after_poll_fires_immediately() {
 
     assert!(queue.is_woken());
 
-    let entry = assert_ready_ok!(poll!(queue)).into_inner();
+    let entry = assert_ready_some!(poll!(queue)).into_inner();
     assert_eq!(entry, "bar");
 }
 
@@ -640,6 +644,175 @@ async fn delay_queue_poll_expired_when_empty() {
     assert!(assert_ready!(poll!(delay_queue)).is_none());
 }
 
+#[tokio::test(start_paused = true)]
+async fn compact_expire_empty() {
+    let mut queue = task::spawn(DelayQueue::new());
+
+    let now = Instant::now();
+
+    queue.insert_at("foo1", now + ms(10));
+    queue.insert_at("foo2", now + ms(10));
+
+    sleep(ms(10)).await;
+
+    let mut res = vec![];
+    while res.len() < 2 {
+        let entry = assert_ready_some!(poll!(queue));
+        res.push(entry.into_inner());
+    }
+
+    queue.compact();
+
+    assert_eq!(queue.len(), 0);
+    assert_eq!(queue.capacity(), 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn compact_remove_empty() {
+    let mut queue = task::spawn(DelayQueue::new());
+
+    let now = Instant::now();
+
+    let key1 = queue.insert_at("foo1", now + ms(10));
+    let key2 = queue.insert_at("foo2", now + ms(10));
+
+    queue.remove(&key1);
+    queue.remove(&key2);
+
+    queue.compact();
+
+    assert_eq!(queue.len(), 0);
+    assert_eq!(queue.capacity(), 0);
+}
+
+#[tokio::test(start_paused = true)]
+// Trigger a re-mapping of keys in the slab due to a `compact` call and
+// test removal of re-mapped keys
+async fn compact_remove_remapped_keys() {
+    let mut queue = task::spawn(DelayQueue::new());
+
+    let now = Instant::now();
+
+    queue.insert_at("foo1", now + ms(10));
+    queue.insert_at("foo2", now + ms(10));
+
+    // should be assigned indices 3 and 4
+    let key3 = queue.insert_at("foo3", now + ms(20));
+    let key4 = queue.insert_at("foo4", now + ms(20));
+
+    sleep(ms(10)).await;
+
+    let mut res = vec![];
+    while res.len() < 2 {
+        let entry = assert_ready_some!(poll!(queue));
+        res.push(entry.into_inner());
+    }
+
+    // items corresponding to `foo3` and `foo4` will be assigned
+    // new indices here
+    queue.compact();
+
+    queue.insert_at("foo5", now + ms(10));
+
+    // test removal of re-mapped keys
+    let expired3 = queue.remove(&key3);
+    let expired4 = queue.remove(&key4);
+
+    assert_eq!(expired3.into_inner(), "foo3");
+    assert_eq!(expired4.into_inner(), "foo4");
+
+    queue.compact();
+    assert_eq!(queue.len(), 1);
+    assert_eq!(queue.capacity(), 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn compact_change_deadline() {
+    let mut queue = task::spawn(DelayQueue::new());
+
+    let mut now = Instant::now();
+
+    queue.insert_at("foo1", now + ms(10));
+    queue.insert_at("foo2", now + ms(10));
+
+    // should be assigned indices 3 and 4
+    queue.insert_at("foo3", now + ms(20));
+    let key4 = queue.insert_at("foo4", now + ms(20));
+
+    sleep(ms(10)).await;
+
+    let mut res = vec![];
+    while res.len() < 2 {
+        let entry = assert_ready_some!(poll!(queue));
+        res.push(entry.into_inner());
+    }
+
+    // items corresponding to `foo3` and `foo4` should be assigned
+    // new indices
+    queue.compact();
+
+    now = Instant::now();
+
+    queue.insert_at("foo5", now + ms(10));
+    let key6 = queue.insert_at("foo6", now + ms(10));
+
+    queue.reset_at(&key4, now + ms(20));
+    queue.reset_at(&key6, now + ms(20));
+
+    // foo3 and foo5 will expire
+    sleep(ms(10)).await;
+
+    while res.len() < 4 {
+        let entry = assert_ready_some!(poll!(queue));
+        res.push(entry.into_inner());
+    }
+
+    sleep(ms(10)).await;
+
+    while res.len() < 6 {
+        let entry = assert_ready_some!(poll!(queue));
+        res.push(entry.into_inner());
+    }
+
+    let entry = assert_ready!(poll!(queue));
+    assert!(entry.is_none());
+}
+
+#[tokio::test(start_paused = true)]
+async fn remove_after_compact() {
+    let now = Instant::now();
+    let mut queue = DelayQueue::new();
+
+    let foo_key = queue.insert_at("foo", now + ms(10));
+    queue.insert_at("bar", now + ms(20));
+    queue.remove(&foo_key);
+    queue.compact();
+
+    let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+        queue.remove(&foo_key);
+    }));
+    assert!(panic.is_err());
+}
+
+#[tokio::test(start_paused = true)]
+async fn remove_after_compact_poll() {
+    let now = Instant::now();
+    let mut queue = task::spawn(DelayQueue::new());
+
+    let foo_key = queue.insert_at("foo", now + ms(10));
+    queue.insert_at("bar", now + ms(20));
+
+    sleep(ms(10)).await;
+    assert_eq!(assert_ready_some!(poll!(queue)).key(), foo_key);
+
+    queue.compact();
+
+    let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+        queue.remove(&foo_key);
+    }));
+    assert!(panic.is_err());
+}
+
 fn ms(n: u64) -> Duration {
     Duration::from_millis(n)
 }

tokio/CHANGELOG.md

@@ -1,3 +1,356 @@
+# 1.19.1 (June 5, 2022)
+
+This release fixes a bug in `Notified::enable`. ([#4747])
+
+[#4747]: https://github.com/tokio-rs/tokio/pull/4747
+
+# 1.19.0 (June 3, 2022)
+
+### Added
+
+- runtime: add `is_finished` method for `JoinHandle` and `AbortHandle` ([#4709])
+- runtime: make global queue and event polling intervals configurable ([#4671])
+- sync: add `Notified::enable` ([#4705])
+- sync: add `watch::Sender::send_if_modified` ([#4591])
+- sync: add resubscribe method to broadcast::Receiver ([#4607])
+- net: add `take_error` to `TcpSocket` and `TcpStream` ([#4739])
+
+### Changed
+
+- io: refactor out usage of Weak in the io handle ([#4656])
+
+### Fixed
+
+- macros: avoid starvation in `join!` and `try_join!` ([#4624])
+
+### Documented
+
+- runtime: clarify semantics of tasks outliving `block_on` ([#4729])
+- time: fix example for `MissedTickBehavior::Burst` ([#4713])
+
+### Unstable
+
+- metrics: correctly update atomics in `IoDriverMetrics` ([#4725])
+- metrics: fix compilation with unstable, process, and rt, but without net ([#4682])
+- task: add `#[track_caller]` to `JoinSet`/`JoinMap` ([#4697])
+- task: add `Builder::{spawn_on, spawn_local_on, spawn_blocking_on}` ([#4683])
+- task: add `consume_budget` for cooperative scheduling ([#4498])
+- task: add `join_set::Builder` for configuring `JoinSet` tasks ([#4687])
+- task: update return value of `JoinSet::join_one` ([#4726])
+
+[#4498]: https://github.com/tokio-rs/tokio/pull/4498
+[#4591]: https://github.com/tokio-rs/tokio/pull/4591
+[#4607]: https://github.com/tokio-rs/tokio/pull/4607
+[#4624]: https://github.com/tokio-rs/tokio/pull/4624
+[#4656]: https://github.com/tokio-rs/tokio/pull/4656
+[#4671]: https://github.com/tokio-rs/tokio/pull/4671
+[#4682]: https://github.com/tokio-rs/tokio/pull/4682
+[#4683]: https://github.com/tokio-rs/tokio/pull/4683
+[#4687]: https://github.com/tokio-rs/tokio/pull/4687
+[#4697]: https://github.com/tokio-rs/tokio/pull/4697
+[#4705]: https://github.com/tokio-rs/tokio/pull/4705
+[#4709]: https://github.com/tokio-rs/tokio/pull/4709
+[#4713]: https://github.com/tokio-rs/tokio/pull/4713
+[#4725]: https://github.com/tokio-rs/tokio/pull/4725
+[#4726]: https://github.com/tokio-rs/tokio/pull/4726
+[#4729]: https://github.com/tokio-rs/tokio/pull/4729
+[#4739]: https://github.com/tokio-rs/tokio/pull/4739
+
+# 1.18.2 (May 5, 2022)
+
+Add missing features for the `winapi` dependency. ([#4663])
+
+[#4663]: https://github.com/tokio-rs/tokio/pull/4663
+
+# 1.18.1 (May 2, 2022)
+
+The 1.18.0 release broke the build for targets without 64-bit atomics when
+building with `tokio_unstable`. This release fixes that. ([#4649])
+
+[#4649]: https://github.com/tokio-rs/tokio/pull/4649
+
+# 1.18.0 (April 27, 2022)
+
+This release adds a number of new APIs in `tokio::net`, `tokio::signal`, and
+`tokio::sync`. In addition, it adds new unstable APIs to `tokio::task` (`Id`s
+for uniquely identifying a task, and `AbortHandle` for remotely cancelling a
+task), as well as a number of bugfixes.
+
+### Fixed
+
+- blocking: add missing `#[track_caller]` for `spawn_blocking` ([#4616])
+- macros: fix `select` macro to process 64 branches ([#4519])
+- net: fix `try_io` methods not calling Mio's `try_io` internally ([#4582])
+- runtime: recover when OS fails to spawn a new thread ([#4485])
+
+### Added
+
+- net: add `UdpSocket::peer_addr` ([#4611])
+- net: add `try_read_buf` method for named pipes ([#4626])
+- signal: add `SignalKind` `Hash`/`Eq` impls and `c_int` conversion ([#4540])
+- signal: add support for signals up to `SIGRTMAX` ([#4555])
+- sync: add `watch::Sender::send_modify` method ([#4310])
+- sync: add `broadcast::Receiver::len` method ([#4542])
+- sync: add `watch::Receiver::same_channel` method ([#4581])
+- sync: implement `Clone` for `RecvError` types ([#4560])
+
+### Changed
+
+- update `mio` to 0.8.1 ([#4582])
+- macros: rename `tokio::select!`'s internal `util` module ([#4543])
+- runtime: use `Vec::with_capacity` when building runtime ([#4553])
+
+### Documented
+
+- improve docs for `tokio_unstable` ([#4524])
+- runtime: include more documentation for thread_pool/worker ([#4511])
+- runtime: update `Handle::current`'s docs to mention `EnterGuard` ([#4567])
+- time: clarify platform specific timer resolution ([#4474])
+- signal: document that `Signal::recv` is cancel-safe ([#4634])
+- sync: `UnboundedReceiver` close docs ([#4548])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`:
+
+- task: add `task::Id` type ([#4630])
+- task: add `AbortHandle` type for cancelling tasks in a `JoinSet` ([#4530],
+  [#4640])
+- task: fix missing `doc(cfg(...))` attributes for `JoinSet` ([#4531])
+- task: fix broken link in `AbortHandle` RustDoc ([#4545])
+- metrics: add initial IO driver metrics ([#4507])
+
+
+[#4616]: https://github.com/tokio-rs/tokio/pull/4616
+[#4519]: https://github.com/tokio-rs/tokio/pull/4519
+[#4582]: https://github.com/tokio-rs/tokio/pull/4582
+[#4485]: https://github.com/tokio-rs/tokio/pull/4485
+[#4613]: https://github.com/tokio-rs/tokio/pull/4613
+[#4611]: https://github.com/tokio-rs/tokio/pull/4611
+[#4626]: https://github.com/tokio-rs/tokio/pull/4626
+[#4540]: https://github.com/tokio-rs/tokio/pull/4540
+[#4555]: https://github.com/tokio-rs/tokio/pull/4555
+[#4310]: https://github.com/tokio-rs/tokio/pull/4310
+[#4542]: https://github.com/tokio-rs/tokio/pull/4542
+[#4581]: https://github.com/tokio-rs/tokio/pull/4581
+[#4560]: https://github.com/tokio-rs/tokio/pull/4560
+[#4631]: https://github.com/tokio-rs/tokio/pull/4631
+[#4582]: https://github.com/tokio-rs/tokio/pull/4582
+[#4543]: https://github.com/tokio-rs/tokio/pull/4543
+[#4553]: https://github.com/tokio-rs/tokio/pull/4553
+[#4524]: https://github.com/tokio-rs/tokio/pull/4524
+[#4511]: https://github.com/tokio-rs/tokio/pull/4511
+[#4567]: https://github.com/tokio-rs/tokio/pull/4567
+[#4474]: https://github.com/tokio-rs/tokio/pull/4474
+[#4634]: https://github.com/tokio-rs/tokio/pull/4634
+[#4548]: https://github.com/tokio-rs/tokio/pull/4548
+[#4630]: https://github.com/tokio-rs/tokio/pull/4630
+[#4530]: https://github.com/tokio-rs/tokio/pull/4530
+[#4640]: https://github.com/tokio-rs/tokio/pull/4640
+[#4531]: https://github.com/tokio-rs/tokio/pull/4531
+[#4545]: https://github.com/tokio-rs/tokio/pull/4545
+[#4507]: https://github.com/tokio-rs/tokio/pull/4507
+
+# 1.17.0 (February 16, 2022)
+
+This release updates the minimum supported Rust version (MSRV) to 1.49, the
+`mio` dependency to v0.8, and the (optional) `parking_lot` dependency to v0.12.
+Additionally, it contains several bug fixes, as well as internal refactoring and
+performance improvements.
+
+### Fixed
+
+- time: prevent panicking in `sleep` with large durations ([#4495])
+- time: eliminate potential panics in `Instant` arithmetic on platforms where
+  `Instant::now` is not monotonic ([#4461])
+- io: fix `DuplexStream` not participating in cooperative yielding ([#4478])
+- rt: fix potential double panic when dropping a `JoinHandle` ([#4430])
+
+### Changed
+
+- update minimum supported Rust version to 1.49 ([#4457])
+- update `parking_lot` dependency to v0.12.0 ([#4459])
+- update `mio` dependency to v0.8 ([#4449])
+- rt: remove an unnecessary lock in the blocking pool ([#4436])
+- rt: remove an unnecessary enum in the basic scheduler ([#4462])
+- time: use bit manipulation instead of modulo to improve performance ([#4480])
+- net: use `std::future::Ready` instead of our own `Ready` future ([#4271])
+- replace deprecated `atomic::spin_loop_hint` with `hint::spin_loop` ([#4491])
+- fix miri failures in intrusive linked lists ([#4397]) 
+
+### Documented
+
+- io: add an example for `tokio::process::ChildStdin` ([#4479])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`:
+
+- task: fix missing location information in `tracing` spans generated by
+  `spawn_local` ([#4483])
+- task: add `JoinSet` for managing sets of tasks ([#4335])
+- metrics: fix compilation error on MIPS ([#4475])
+- metrics: fix compilation error on arm32v7 ([#4453])
+
+[#4495]: https://github.com/tokio-rs/tokio/pull/4495
+[#4461]: https://github.com/tokio-rs/tokio/pull/4461
+[#4478]: https://github.com/tokio-rs/tokio/pull/4478
+[#4430]: https://github.com/tokio-rs/tokio/pull/4430
+[#4457]: https://github.com/tokio-rs/tokio/pull/4457
+[#4459]: https://github.com/tokio-rs/tokio/pull/4459
+[#4449]: https://github.com/tokio-rs/tokio/pull/4449
+[#4462]: https://github.com/tokio-rs/tokio/pull/4462
+[#4436]: https://github.com/tokio-rs/tokio/pull/4436
+[#4480]: https://github.com/tokio-rs/tokio/pull/4480
+[#4271]: https://github.com/tokio-rs/tokio/pull/4271
+[#4491]: https://github.com/tokio-rs/tokio/pull/4491
+[#4397]: https://github.com/tokio-rs/tokio/pull/4397
+[#4479]: https://github.com/tokio-rs/tokio/pull/4479
+[#4483]: https://github.com/tokio-rs/tokio/pull/4483
+[#4335]: https://github.com/tokio-rs/tokio/pull/4335
+[#4475]: https://github.com/tokio-rs/tokio/pull/4475
+[#4453]: https://github.com/tokio-rs/tokio/pull/4453
+
+# 1.16.1 (January 28, 2022)
+
+This release fixes a bug in [#4428] with the change [#4437].
+
+[#4428]: https://github.com/tokio-rs/tokio/pull/4428
+[#4437]: https://github.com/tokio-rs/tokio/pull/4437
+
+# 1.16.0 (January 27, 2022)
+
+Fixes a soundness bug in `io::Take` ([#4428]). The unsoundness is exposed when
+leaking memory in the given `AsyncRead` implementation and then overwriting the
+supplied buffer:
+
+```rust
+impl AsyncRead for Buggy {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>
+    ) -> Poll<Result<()>> {
+      let new_buf = vec![0; 5].leak();
+      *buf = ReadBuf::new(new_buf);
+      buf.put_slice(b"hello");
+      Poll::Ready(Ok(()))
+    }
+}
+```
+
+Also, this release includes improvements to the multi-threaded scheduler that
+can increase throughput by up to 20% in some cases ([#4383]).
+
+### Fixed
+
+- io: **soundness** don't expose uninitialized memory when using `io::Take` in edge case ([#4428])
+- fs: ensure `File::write` results in a `write` syscall when the runtime shuts down ([#4316])
+- process: drop pipe after child exits in `wait_with_output` ([#4315])
+- rt: improve error message when spawning a thread fails ([#4398])
+- rt: reduce false-positive thread wakups in the multi-threaded scheduler ([#4383])
+- sync: don't inherit `Send` from `parking_lot::*Guard` ([#4359])
+
+### Added
+
+- net: `TcpSocket::linger()` and `set_linger()` ([#4324])
+- net: impl `UnwindSafe` for socket types ([#4384])
+- rt: impl `UnwindSafe` for `JoinHandle` ([#4418])
+- sync: `watch::Receiver::has_changed()` ([#4342])
+- sync: `oneshot::Receiver::blocking_recv()` ([#4334])
+- sync: `RwLock` blocking operations ([#4425])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`
+
+- rt: **breaking change** overhaul runtime metrics API ([#4373])
+
+[#4428]: https://github.com/tokio-rs/tokio/pull/4428
+[#4316]: https://github.com/tokio-rs/tokio/pull/4316
+[#4315]: https://github.com/tokio-rs/tokio/pull/4315
+[#4398]: https://github.com/tokio-rs/tokio/pull/4398
+[#4383]: https://github.com/tokio-rs/tokio/pull/4383
+[#4359]: https://github.com/tokio-rs/tokio/pull/4359
+[#4324]: https://github.com/tokio-rs/tokio/pull/4324
+[#4384]: https://github.com/tokio-rs/tokio/pull/4384
+[#4418]: https://github.com/tokio-rs/tokio/pull/4418
+[#4342]: https://github.com/tokio-rs/tokio/pull/4342
+[#4334]: https://github.com/tokio-rs/tokio/pull/4334
+[#4425]: https://github.com/tokio-rs/tokio/pull/4425
+[#4373]: https://github.com/tokio-rs/tokio/pull/4373
+
+# 1.15.0 (December 15, 2021)
+
+### Fixed
+
+- io: add cooperative yielding support to `io::empty()` ([#4300])
+- time: make timeout robust against budget-depleting tasks ([#4314])
+
+### Changed
+
+- update minimum supported Rust version to 1.46.
+
+### Added
+
+- time: add `Interval::reset()` ([#4248])
+- io: add explicit lifetimes to `AsyncFdReadyGuard` ([#4267])
+- process: add `Command::as_std()` ([#4295])
+
+### Added (unstable)
+
+- tracing: instrument `tokio::sync` types ([#4302])
+
+[#4302]: https://github.com/tokio-rs/tokio/pull/4302
+[#4300]: https://github.com/tokio-rs/tokio/pull/4300
+[#4295]: https://github.com/tokio-rs/tokio/pull/4295
+[#4267]: https://github.com/tokio-rs/tokio/pull/4267
+[#4248]: https://github.com/tokio-rs/tokio/pull/4248
+[#4314]: https://github.com/tokio-rs/tokio/pull/4314
+
+# 1.14.0 (November 15, 2021)
+
+### Fixed
+
+- macros: fix compiler errors when using `mut` patterns in `select!` ([#4211])
+- sync: fix a data race between `oneshot::Sender::send` and awaiting a
+  `oneshot::Receiver` when the oneshot has been closed ([#4226])
+- sync: make `AtomicWaker` panic safe ([#3689])
+- runtime: fix basic scheduler dropping tasks outside a runtime context
+  ([#4213])
+
+### Added
+
+- stats: add `RuntimeStats::busy_duration_total` ([#4179], [#4223])
+
+### Changed
+
+- io: updated `copy` buffer size to match `std::io::copy` ([#4209])
+
+### Documented
+
+-  io: rename buffer to file in doc-test ([#4230])
+-  sync: fix Notify example ([#4212])
+
+[#4211]: https://github.com/tokio-rs/tokio/pull/4211
+[#4226]: https://github.com/tokio-rs/tokio/pull/4226
+[#3689]: https://github.com/tokio-rs/tokio/pull/3689
+[#4213]: https://github.com/tokio-rs/tokio/pull/4213
+[#4179]: https://github.com/tokio-rs/tokio/pull/4179
+[#4223]: https://github.com/tokio-rs/tokio/pull/4223
+[#4209]: https://github.com/tokio-rs/tokio/pull/4209
+[#4230]: https://github.com/tokio-rs/tokio/pull/4230
+[#4212]: https://github.com/tokio-rs/tokio/pull/4212
+
+# 1.13.1 (November 15, 2021)
+
+### Fixed
+
+- sync: fix a data race between `oneshot::Sender::send` and awaiting a
+  `oneshot::Receiver` when the oneshot has been closed ([#4226])
+
+[#4226]: https://github.com/tokio-rs/tokio/pull/4226
+
 # 1.13.0 (October 29, 2021)
 
 ### Fixed

tokio/Cargo.toml

@@ -3,16 +3,15 @@ name = "tokio"
 # When releasing to crates.io:
 # - Remove path dependencies
 # - Update doc url
-#   - Cargo.toml
 #   - README.md
 # - Update CHANGELOG.md.
 # - Create "v1.0.x" git tag.
-version = "1.13.0"
+version = "1.19.1"
 edition = "2018"
+rust-version = "1.49"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 readme = "README.md"
-documentation = "https://docs.rs/tokio/1.13.0/tokio/"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
 description = """
@@ -47,14 +46,12 @@ io-util = ["memchr", "bytes"]
 # stdin, stdout, stderr
 io-std = []
 macros = ["tokio-macros"]
-stats = []
 net = [
   "libc",
   "mio/os-poll",
-  "mio/os-util",
-  "mio/tcp",
-  "mio/udp",
-  "mio/uds",
+  "mio/os-ext",
+  "mio/net",
+  "socket2",
   "winapi/namedpipeapi",
 ]
 process = [
@@ -62,13 +59,13 @@ process = [
   "once_cell",
   "libc",
   "mio/os-poll",
-  "mio/os-util",
-  "mio/uds",
+  "mio/os-ext",
+  "mio/net",
   "signal-hook-registry",
   "winapi/threadpoollegacyapiset",
 ]
 # Includes basic task execution capabilities
-rt = []
+rt = ["once_cell"]
 rt-multi-thread = [
   "num_cpus",
   "rt",
@@ -77,8 +74,8 @@ signal = [
   "once_cell",
   "libc",
   "mio/os-poll",
-  "mio/uds",
-  "mio/os-util",
+  "mio/net",
+  "mio/os-ext",
   "signal-hook-registry",
   "winapi/consoleapi",
 ]
@@ -86,8 +83,13 @@ sync = []
 test-util = ["rt", "sync", "time"]
 time = []
 
+# Technically, removing this is a breaking change even though it only ever did
+# anything with the unstable flag on. It is probably safe to get rid of it after
+# a few releases.
+stats = []
+
 [dependencies]
-tokio-macros = { version = "1.1.0", path = "../tokio-macros", optional = true }
+tokio-macros = { version = "1.7.0", path = "../tokio-macros", optional = true }
 
 pin-project-lite = "0.2.0"
 
@@ -95,14 +97,15 @@ pin-project-lite = "0.2.0"
 bytes = { version = "1.0.0", optional = true }
 once_cell = { version = "1.5.2", optional = true }
 memchr = { version = "2.2", optional = true }
-mio = { version = "0.7.6", optional = true }
+mio = { version = "0.8.1", optional = true }
+socket2 = { version = "0.4.4", optional = true, features = [ "all" ] }
 num_cpus = { version = "1.8.0", optional = true }
-parking_lot = { version = "0.11.0", optional = true }
+parking_lot = { version = "0.12.0", optional = true }
 
 # Currently unstable. The API exposed by these features may be broken at any time.
 # Requires `--cfg tokio_unstable` to enable.
 [target.'cfg(tokio_unstable)'.dependencies]
-tracing = { version = "0.1.21", default-features = false, features = ["std"], optional = true } # Not in full
+tracing = { version = "0.1.25", default-features = false, features = ["std"], optional = true } # Not in full
 
 [target.'cfg(unix)'.dependencies]
 libc = { version = "0.2.42", optional = true }
@@ -110,11 +113,12 @@ signal-hook-registry = { version = "1.1.1", optional = true }
 
 [target.'cfg(unix)'.dev-dependencies]
 libc = { version = "0.2.42" }
-nix = { version = "0.22.0" }
+nix = { version = "0.24", default-features = false, features = ["fs", "socket"] }
 
 [target.'cfg(windows)'.dependencies.winapi]
 version = "0.3.8"
 default-features = false
+features = ["std", "winsock2", "mswsock", "handleapi", "ws2ipdef", "ws2tcpip"]
 optional = true
 
 [target.'cfg(windows)'.dev-dependencies.ntapi]
@@ -124,25 +128,31 @@ version = "0.3.6"
 tokio-test = { version = "0.4.0", path = "../tokio-test" }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 futures = { version = "0.3.0", features = ["async-await"] }
-mockall = "0.10.2"
-proptest = "1"
-rand = "0.8.0"
+mockall = "0.11.1"
 tempfile = "3.1.0"
 async-stream = "0.3"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+proptest = "1"
+rand = "0.8.0"
 socket2 = "0.4"
 
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
+wasm-bindgen-test = "0.3.0"
+
 [target.'cfg(target_os = "freebsd")'.dev-dependencies]
 mio-aio = { version = "0.6.0", features = ["tokio"] }
 
 [target.'cfg(loom)'.dev-dependencies]
-loom = { version = "0.5", features = ["futures", "checkpoint"] }
-
-[build-dependencies]
-autocfg = "1" # Needed for conditionally enabling `track-caller`
+loom = { version = "0.5.2", features = ["futures", "checkpoint"] }
 
 [package.metadata.docs.rs]
 all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
+# enable unstable features in the documentation
+rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]
+# it's necessary to _also_ pass `--cfg tokio_unstable` to rustc, or else
+# dependencies will not be enabled, and the docs build will fail.
+rustc-args = ["--cfg", "tokio_unstable"]
 
 [package.metadata.playground]
 features = ["full", "test-util"]

tokio/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2022 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio/README.md

@@ -56,7 +56,7 @@ Make sure you activated the full features of the tokio crate on Cargo.toml:
 
 ```toml
 [dependencies]
-tokio = { version = "1.13.0", features = ["full"] }
+tokio = { version = "1.19.1", features = ["full"] }
 ```
 Then, on your main.rs:
 
@@ -163,9 +163,21 @@ several other libraries, including:
 
 ## Supported Rust Versions
 
-Tokio is built against the latest stable release. The minimum supported version
-is 1.45.  The current Tokio version is not guaranteed to build on Rust versions
-earlier than the minimum supported version.
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- CONTRIBUTING.md
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
+Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
+least** 6 months. When increasing the MSRV, the new Rust version must have been
+released at least six months ago. The current MSRV is 1.49.0.
 
 ## Release schedule
 
@@ -180,17 +192,18 @@ warrants a patch release with a fix for the bug, it will be backported and
 released as a new patch release for each LTS minor version. Our current LTS
 releases are:
 
- * `1.8.x` - LTS release until February 2022.
+ * `1.14.x` - LTS release until June 2022.
+ * `1.18.x` - LTS release until January 2023
 
 Each LTS release will continue to receive backported fixes for at least half a
 year. If you wish to use a fixed minor release in your project, we recommend
 that you use an LTS release.
 
 To use a fixed minor version, you can specify the version with a tilde. For
-example, to specify that you wish to use the newest `1.8.x` patch release, you
+example, to specify that you wish to use the newest `1.14.x` patch release, you
 can use the following dependency specification:
 ```text
-tokio = { version = "~1.8", features = [...] }
+tokio = { version = "~1.14", features = [...] }
 ```
 
 ## License

tokio/build.rs

@@ -1,22 +0,0 @@
-use autocfg::AutoCfg;
-
-fn main() {
-    match AutoCfg::new() {
-        Ok(ac) => {
-            // The #[track_caller] attribute was stabilized in rustc 1.46.0.
-            if ac.probe_rustc_version(1, 46) {
-                autocfg::emit("tokio_track_caller")
-            }
-        }
-
-        Err(e) => {
-            // If we couldn't detect the compiler version and features, just
-            // print a warning. This isn't a fatal error: we can still build
-            // Tokio, we just can't enable cfgs automatically.
-            println!(
-                "cargo:warning=tokio: failed to detect compiler features: {}",
-                e
-            );
-        }
-    }
-}

tokio/src/blocking.rs

@@ -1,5 +1,11 @@
 cfg_rt! {
     pub(crate) use crate::runtime::spawn_blocking;
+
+    cfg_fs! {
+        #[allow(unused_imports)]
+        pub(crate) use crate::runtime::spawn_mandatory_blocking;
+    }
+
     pub(crate) use crate::task::JoinHandle;
 }
 
@@ -16,7 +22,16 @@ cfg_not_rt! {
     {
         assert_send_sync::<JoinHandle<std::cell::Cell<()>>>();
         panic!("requires the `rt` Tokio feature flag")
+    }
 
+    cfg_fs! {
+        pub(crate) fn spawn_mandatory_blocking<F, R>(_f: F) -> Option<JoinHandle<R>>
+        where
+            F: FnOnce() -> R + Send + 'static,
+            R: Send + 'static,
+        {
+            panic!("requires the `rt` Tokio feature flag")
+        }
     }
 
     pub(crate) struct JoinHandle<R> {

tokio/src/coop.rs

@@ -59,13 +59,9 @@ impl Budget {
     const fn unconstrained() -> Budget {
         Budget(None)
     }
-}
 
-cfg_rt_multi_thread! {
-    impl Budget {
-        fn has_remaining(self) -> bool {
-            self.0.map(|budget| budget > 0).unwrap_or(true)
-        }
+    fn has_remaining(self) -> bool {
+        self.0.map(|budget| budget > 0).unwrap_or(true)
     }
 }
 
@@ -107,16 +103,16 @@ fn with_budget<R>(budget: Budget, f: impl FnOnce() -> R) -> R {
     })
 }
 
+#[inline(always)]
+pub(crate) fn has_budget_remaining() -> bool {
+    CURRENT.with(|cell| cell.get().has_remaining())
+}
+
 cfg_rt_multi_thread! {
     /// Sets the current task's budget.
     pub(crate) fn set(budget: Budget) {
         CURRENT.with(|cell| cell.set(budget))
     }
-
-    #[inline(always)]
-    pub(crate) fn has_budget_remaining() -> bool {
-        CURRENT.with(|cell| cell.get().has_remaining())
-    }
 }
 
 cfg_rt! {
@@ -211,6 +207,9 @@ cfg_coop! {
 mod test {
     use super::*;
 
+    #[cfg(target_arch = "wasm32")]
+    use wasm_bindgen_test::wasm_bindgen_test as test;
+
     fn get() -> Budget {
         CURRENT.with(|cell| cell.get())
     }

tokio/src/fs/file.rs

@@ -19,17 +19,17 @@ use std::task::Context;
 use std::task::Poll;
 use std::task::Poll::*;
 
-#[cfg(test)]
-use super::mocks::spawn_blocking;
 #[cfg(test)]
 use super::mocks::JoinHandle;
 #[cfg(test)]
 use super::mocks::MockFile as StdFile;
-#[cfg(not(test))]
-use crate::blocking::spawn_blocking;
+#[cfg(test)]
+use super::mocks::{spawn_blocking, spawn_mandatory_blocking};
 #[cfg(not(test))]
 use crate::blocking::JoinHandle;
 #[cfg(not(test))]
+use crate::blocking::{spawn_blocking, spawn_mandatory_blocking};
+#[cfg(not(test))]
 use std::fs::File as StdFile;
 
 /// A reference to an open file on the filesystem.
@@ -649,7 +649,7 @@ impl AsyncWrite for File {
                     let n = buf.copy_from(src);
                     let std = me.std.clone();
 
-                    inner.state = Busy(spawn_blocking(move || {
+                    let blocking_task_join_handle = spawn_mandatory_blocking(move || {
                         let res = if let Some(seek) = seek {
                             (&*std).seek(seek).and_then(|_| buf.write_to(&mut &*std))
                         } else {
@@ -657,7 +657,12 @@ impl AsyncWrite for File {
                         };
 
                         (Operation::Write(res), buf)
-                    }));
+                    })
+                    .ok_or_else(|| {
+                        io::Error::new(io::ErrorKind::Other, "background task failed")
+                    })?;
+
+                    inner.state = Busy(blocking_task_join_handle);
 
                     return Ready(Ok(n));
                 }

tokio/src/fs/file/tests.rs

@@ -228,6 +228,7 @@ fn flush_while_idle() {
 }
 
 #[test]
+#[cfg_attr(miri, ignore)] // takes a really long time with miri
 fn read_with_buffer_larger_than_max() {
     // Chunks
     let chunk_a = 16 * 1024;
@@ -299,6 +300,7 @@ fn read_with_buffer_larger_than_max() {
 }
 
 #[test]
+#[cfg_attr(miri, ignore)] // takes a really long time with miri
 fn write_with_buffer_larger_than_max() {
     // Chunks
     let chunk_a = 16 * 1024;

tokio/src/fs/mocks.rs

@@ -105,6 +105,21 @@ where
     JoinHandle { rx }
 }
 
+pub(super) fn spawn_mandatory_blocking<F, R>(f: F) -> Option<JoinHandle<R>>
+where
+    F: FnOnce() -> R + Send + 'static,
+    R: Send + 'static,
+{
+    let (tx, rx) = oneshot::channel();
+    let task = Box::new(move || {
+        let _ = tx.send(f());
+    });
+
+    QUEUE.with(|cell| cell.borrow_mut().push_back(task));
+
+    Some(JoinHandle { rx })
+}
+
 impl<T> Future for JoinHandle<T> {
     type Output = Result<T, io::Error>;
 

tokio/src/future/mod.rs

@@ -8,11 +8,6 @@ pub(crate) mod maybe_done;
 mod poll_fn;
 pub use poll_fn::poll_fn;
 
-cfg_not_loom! {
-    mod ready;
-    pub(crate) use ready::{ok, Ready};
-}
-
 cfg_process! {
     mod try_join;
     pub(crate) use try_join::try_join3;