chore: prepare Tokio 1.7.3

Tokio 1.7.0 introduced a change intended to eagerly shutdown newly
spawned tasks if the runtime is in the process of shutting down.
However, it introduced a bug where already spawned tasks could be
shutdown too early, resulting in the potential introduction of deadlocks
if tasks acquired mutexes in drop handlers.

Fixes #3869

.cirrus.yml

@@ -21,9 +21,9 @@ task:
       rustc --version
   test_script:
     - . $HOME/.cargo/env
-    - cargo test --all
+    - cargo test --all --all-features
     - cargo doc --all --no-deps
   i686_test_script:
     - . $HOME/.cargo/env
     - |
-      cargo test --all --target i686-unknown-freebsd
+      cargo test --all --all-features --target i686-unknown-freebsd

.clippy.toml

@@ -0,0 +1 @@
+msrv = "1.45"

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [tokio-rs]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,12 +9,7 @@ assignees: ''
 
 **Version**
 List the versions of all `tokio` crates you are using. The easiest way to get
-this information is using `cargo-tree`.
-
-`cargo install cargo-tree`
-(see install here: https://github.com/sfackler/cargo-tree)
-
-Then:
+this information is using `cargo tree` subcommand:
 
 `cargo tree | grep tokio`
 

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ name: CI
 env:
   RUSTFLAGS: -Dwarnings
   RUST_BACKTRACE: 1
-  nightly: nightly-2020-09-21
+  nightly: nightly-2021-04-25
   minrust: 1.45.2
 
 jobs:
@@ -68,6 +68,13 @@ jobs:
         run: cargo hack test --each-feature
         working-directory: tests-build
 
+      # Build benchmarks. Run of benchmarks is done by bench.yml workflow.
+      - name: build benches
+        run: cargo build --benches
+        working-directory: benches
+        # bench.yml workflow runs benchmarks only on linux.
+        if: startsWith(matrix.os, 'ubuntu')
+
   valgrind:
     name: valgrind
     runs-on: ubuntu-latest
@@ -82,14 +89,23 @@ jobs:
           sudo apt-get install -y valgrind
 
       # Compile tests
-      - name: cargo build
+      - name: cargo build test-mem
         run: cargo build --features rt-net --bin test-mem
         working-directory: tests-integration
 
       # Run with valgrind
-      - name: Run valgrind
+      - name: Run valgrind test-mem
         run: valgrind  --leak-check=full --show-leak-kinds=all ./target/debug/test-mem
 
+      # Compile tests
+      - name: cargo build test-process-signal
+        run: cargo build --features rt-process-signal --bin test-process-signal
+        working-directory: tests-integration
+
+      # Run with valgrind
+      - name: Run valgrind test-process-signal
+        run: valgrind --leak-check=full --show-leak-kinds=all ./target/debug/test-process-signal
+
   test-unstable:
     name: test tokio full --unstable
     runs-on: ${{ matrix.os }}
@@ -222,7 +238,7 @@ jobs:
           cargo hack --remove-dev-deps --workspace
           # Update Cargo.lock to minimal version dependencies.
           cargo update -Z minimal-versions
-          cargo check --all-features
+          cargo hack check --all-features --ignore-private
 
   fmt:
     name: fmt
@@ -249,13 +265,13 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install Rust
-        run: rustup update ${{ env.minrust }} && rustup default ${{ env.minrust }}
+        run: rustup update 1.52.1 && rustup default 1.52.1
       - name: Install clippy
         run: rustup component add clippy
 
       # Run clippy
       - name: "clippy --all"
-        run: cargo clippy --all --tests
+        run: cargo clippy --all --tests --all-features
 
   docs:
     name: docs
@@ -268,9 +284,9 @@ jobs:
           override: true
 
       - name: "doc --lib --all-features"
-        run: cargo doc --lib --no-deps --all-features
+        run: cargo doc --lib --no-deps --all-features --document-private-items
         env:
-          RUSTDOCFLAGS: --cfg docsrs
+          RUSTDOCFLAGS: --cfg docsrs -Dwarnings
 
   loom:
     name: loom

CONTRIBUTING.md

@@ -124,17 +124,27 @@ arguments to many common cargo commands. This section lists some commonly needed
 commands.
 
 Some commands just need the `--all-features` argument:
+
 ```
 cargo build --all-features
 cargo check --all-features
 cargo test --all-features
 ```
+
 When building documentation normally, the markers that list the features
 required for various parts of Tokio are missing. To build the documentation
 correctly, use this command:
+
 ```
 RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features
 ```
+
+There is currently a [bug in cargo] that means documentation cannot be built
+from the root of the workspace. If you `cd` into the `tokio` subdirectory the
+command shown above will work.
+
+[bug in cargo]: https://github.com/rust-lang/cargo/issues/9274
+
 The `cargo fmt` command does not work on the Tokio codebase. You can use the
 command below instead:
 
@@ -427,11 +437,14 @@ _Adapted from the [Node.js contributing guide][node]_.
 
 ## Keeping track of issues and PRs
 
-The Tokio GitHub repository has a lot of issues and PRs, which is not easy to
-keep track of. This section explains the meaning of various labels, as well as
-our [GitHub project][project]. The section is primarily targeted at maintainers.
+The Tokio GitHub repository has a lot of issues and PRs to keep track of. This
+section explains the meaning of various labels, as well as our [GitHub
+project][project]. The section is primarily targeted at maintainers.  Most
+contributors aren't able to set these labels.
 
-**Area.** The area label describes the crates relevant to this issue or PR.
+### Area
+
+The area label describes the crates relevant to this issue or PR.
 
  - **A-tokio** This issue concerns the main Tokio crate.
  - **A-tokio-util** This issue concerns the `tokio-util` crate.
@@ -442,7 +455,7 @@ our [GitHub project][project]. The section is primarily targeted at maintainers.
    be used for the procedural macros, and not `join!` or `select!`.
  - **A-ci** This issue concerns our GitHub Actions setup.
 
-**Category.** The category label describes the category.
+### Category
 
  - **C-bug** This is a bug-report. Bug-fix PRs use `C-enhancement` instead.
  - **C-enhancement** This is a PR that adds a new features.
@@ -460,8 +473,7 @@ our [GitHub project][project]. The section is primarily targeted at maintainers.
  - **C-request** A non-feature request, e.g. "please add deprecation notices to
    `-alpha.*` versions of crates"
 
-**Call for participation.** I don't know why it's called `E-`. Many issues are
-missing a difficulty rating, and you should feel free to add one.
+### Calls for participation
 
  - **E-help-wanted** Stuff where we want help. Often seen together with `C-bug`
    or `C-feature-accepted`.
@@ -473,7 +485,13 @@ missing a difficulty rating, and you should feel free to add one.
  - **E-needs-mvce** This bug is missing a minimal complete and verifiable
    example.
 
-**Module.** A more fine groaned categorization than area.
+The "E-" prefix is the same as used in the Rust compiler repository. Some
+issues are missing a difficulty rating, but feel free to ask on our Discord
+server if you want to know how difficult an issue likely is.
+
+### Module
+
+The module label provides a more fine grained categorization than **Area**.
 
  - **M-blocking** Things relevant to `spawn_blocking`, `block_in_place`.
  - **M-codec** The `tokio_util::codec` module.
@@ -491,7 +509,9 @@ missing a difficulty rating, and you should feel free to add one.
  - **M-time** The `tokio::time` module.
  - **M-tracing** Tracing support in Tokio.
 
-**Topic.** Some extra information.
+### Topic
+
+Some extra information.
 
  - **T-docs** This is about documentation.
  - **T-performance** This is about performance.
@@ -501,6 +521,34 @@ Any label not listed here is not in active use.
 
 [project]: https://github.com/orgs/tokio-rs/projects/1
 
+## LTS guarantees
+
+Tokio ≥1.0.0 comes with LTS guarantees:
+
+ * A minimum of 5 years of maintenance.
+ * A minimum of 3 years before a hypothetical 2.0 release.
+
+The goal of these guarantees is to provide stability to the ecosystem.
+
+## Mininum Supported Rust Version (MSRV)
+
+ * All Tokio ≥1.0.0 releases will support at least a 6-month old Rust
+   compiler release.
+ * The MSRV will only be increased on 1.x releases.
+
+## Versioning Policy
+
+With Tokio ≥1.0.0:
+
+ * Patch (1.\_.x) releases _should only_ contain bug fixes or documentation
+   changes. Besides this, these releases should not substantially change
+   runtime behavior.
+ * Minor (1.x) releases may contain new functionality, MSRV increases (see
+   above), minor dependency updates, deprecations, and larger internal
+   implementation changes.
+
+This is as defined by [Semantic Versioning 2.0](https://semver.org/).
+
 ## Releasing
 
 Since the Tokio project consists of a number of crates, many of which depend on
@@ -533,9 +581,8 @@ When releasing a new version of a crate, follow these steps:
 2. **Update Cargo metadata.** After releasing any path dependencies, update the
    `version` field in `Cargo.toml` to the new version, and the `documentation`
    field to the docs.rs URL of the new version.
-3. **Update other documentation links.** Update the `#![doc(html_root_url)]`
-   attribute in the crate's `lib.rs` and the "Documentation" link in the crate's
-   `README.md` to point to the docs.rs URL of the new version.
+3. **Update other documentation links.** Update the "Documentation" link in the
+   crate's `README.md` to point to the docs.rs URL of the new version.
 4. **Update the changelog for the crate.** Each crate in the Tokio repository
    has its own `CHANGELOG.md` in that crate's subdirectory. Any changes to that
    crate since the last release should be added to the changelog. Change

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2019 Tokio Contributors
+Copyright (c) 2021 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

README.md

@@ -50,7 +50,15 @@ an asynchronous application.
 
 ## Example
 
-A basic TCP echo server with Tokio:
+A basic TCP echo server with Tokio.
+
+Make sure you activated the full features of the tokio crate on Cargo.toml:
+
+```toml
+[dependencies]
+tokio = { version = "1.7.0", features = ["full"] }
+```
+Then, on your main.rs:
 
 ```rust,no_run
 use tokio::net::TcpListener;
@@ -58,7 +66,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let mut listener = TcpListener::bind("127.0.0.1:8080").await?;
+    let listener = TcpListener::bind("127.0.0.1:8080").await?;
 
     loop {
         let (mut socket, _) = listener.accept().await?;

benches/Cargo.toml

@@ -5,9 +5,13 @@ publish = false
 edition = "2018"
 
 [dependencies]
-tokio = { version = "1.0.0", path = "../tokio", features = ["full"] }
+tokio = { version = "1.5.0", path = "../tokio", features = ["full"] }
 bencher = "0.1.5"
 
+[dev-dependencies]
+tokio-util = { version = "0.6.6", path = "../tokio-util", features = ["full"] }
+tokio-stream = { path = "../tokio-stream" }
+
 [target.'cfg(unix)'.dependencies]
 libc = "0.2.42"
 
@@ -41,3 +45,8 @@ harness = false
 name = "signal"
 path = "signal.rs"
 harness = false
+
+[[bench]]
+name = "fs"
+path = "fs.rs"
+harness = false

benches/fs.rs

@@ -0,0 +1,103 @@
+#![cfg(unix)]
+
+use tokio_stream::StreamExt;
+
+use tokio::fs::File;
+use tokio::io::AsyncReadExt;
+use tokio_util::codec::{BytesCodec, FramedRead /*FramedWrite*/};
+
+use bencher::{benchmark_group, benchmark_main, Bencher};
+
+use std::fs::File as StdFile;
+use std::io::Read as StdRead;
+
+fn rt() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(2)
+        .build()
+        .unwrap()
+}
+
+const BLOCK_COUNT: usize = 1_000;
+
+const BUFFER_SIZE: usize = 4096;
+const DEV_ZERO: &'static str = "/dev/zero";
+
+fn async_read_codec(b: &mut Bencher) {
+    let rt = rt();
+
+    b.iter(|| {
+        let task = || async {
+            let file = File::open(DEV_ZERO).await.unwrap();
+            let mut input_stream = FramedRead::with_capacity(file, BytesCodec::new(), BUFFER_SIZE);
+
+            for _i in 0..BLOCK_COUNT {
+                let _bytes = input_stream.next().await.unwrap();
+            }
+        };
+
+        rt.block_on(task());
+    });
+}
+
+fn async_read_buf(b: &mut Bencher) {
+    let rt = rt();
+
+    b.iter(|| {
+        let task = || async {
+            let mut file = File::open(DEV_ZERO).await.unwrap();
+            let mut buffer = [0u8; BUFFER_SIZE];
+
+            for _i in 0..BLOCK_COUNT {
+                let count = file.read(&mut buffer).await.unwrap();
+                if count == 0 {
+                    break;
+                }
+            }
+        };
+
+        rt.block_on(task());
+    });
+}
+
+fn async_read_std_file(b: &mut Bencher) {
+    let rt = rt();
+
+    let task = || async {
+        let mut file = tokio::task::block_in_place(|| Box::pin(StdFile::open(DEV_ZERO).unwrap()));
+
+        for _i in 0..BLOCK_COUNT {
+            let mut buffer = [0u8; BUFFER_SIZE];
+            let mut file_ref = file.as_mut();
+
+            tokio::task::block_in_place(move || {
+                file_ref.read_exact(&mut buffer).unwrap();
+            });
+        }
+    };
+
+    b.iter(|| {
+        rt.block_on(task());
+    });
+}
+
+fn sync_read(b: &mut Bencher) {
+    b.iter(|| {
+        let mut file = StdFile::open(DEV_ZERO).unwrap();
+        let mut buffer = [0u8; BUFFER_SIZE];
+
+        for _i in 0..BLOCK_COUNT {
+            file.read_exact(&mut buffer).unwrap();
+        }
+    });
+}
+
+benchmark_group!(
+    file,
+    async_read_std_file,
+    async_read_buf,
+    async_read_codec,
+    sync_read
+);
+
+benchmark_main!(file);

examples/Cargo.toml

@@ -7,11 +7,10 @@ edition = "2018"
 # If you copy one of the examples into a new project, you should be using
 # [dependencies] instead.
 [dev-dependencies]
-tokio = { version = "1.0.0", path = "../tokio", features = ["full", "tracing"] }
-tokio-util = { version = "0.6.0", path = "../tokio-util", features = ["full"] }
+tokio = { version = "1.0.0", path = "../tokio",features = ["full", "tracing"] }
+tokio-util = { version = "0.6.3", path = "../tokio-util",features = ["full"] }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 
-async-stream = "0.3"
 tracing = "0.1"
 tracing-subscriber = { version = "0.2.7", default-features = false, features = ["fmt", "ansi", "env-filter", "chrono", "tracing-log"] }
 bytes = "1.0.0"
@@ -23,7 +22,10 @@ serde_json = "1.0"
 httparse = "1.0"
 time = "0.1"
 once_cell = "1.5.2"
+rand = "0.8.3"
 
+[target.'cfg(windows)'.dev-dependencies.winapi]
+version = "0.3.8"
 
 [[example]]
 name = "chat"
@@ -77,3 +79,11 @@ path = "custom-executor.rs"
 [[example]]
 name = "custom-executor-tokio-context"
 path = "custom-executor-tokio-context.rs"
+
+[[example]]
+name = "named-pipe"
+path = "named-pipe.rs"
+
+[[example]]
+name = "named-pipe-multi-client"
+path = "named-pipe-multi-client.rs"

examples/chat.rs

@@ -28,8 +28,8 @@
 
 use tokio::net::{TcpListener, TcpStream};
 use tokio::sync::{mpsc, Mutex};
-use tokio_stream::{Stream, StreamExt};
-use tokio_util::codec::{Framed, LinesCodec, LinesCodecError};
+use tokio_stream::StreamExt;
+use tokio_util::codec::{Framed, LinesCodec};
 
 use futures::SinkExt;
 use std::collections::HashMap;
@@ -37,9 +37,7 @@ use std::env;
 use std::error::Error;
 use std::io;
 use std::net::SocketAddr;
-use std::pin::Pin;
 use std::sync::Arc;
-use std::task::{Context, Poll};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn Error>> {
@@ -101,6 +99,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
 /// Shorthand for the transmit half of the message channel.
 type Tx = mpsc::UnboundedSender<String>;
 
+/// Shorthand for the receive half of the message channel.
+type Rx = mpsc::UnboundedReceiver<String>;
+
 /// Data that is shared between all peers in the chat server.
 ///
 /// This is the set of `Tx` handles for all connected clients. Whenever a
@@ -124,7 +125,7 @@ struct Peer {
     ///
     /// This is used to receive messages from peers. When a message is received
     /// off of this `Rx`, it will be written to the socket.
-    rx: Pin<Box<dyn Stream<Item = String> + Send>>,
+    rx: Rx,
 }
 
 impl Shared {
@@ -156,58 +157,15 @@ impl Peer {
         let addr = lines.get_ref().peer_addr()?;
 
         // Create a channel for this peer
-        let (tx, mut rx) = mpsc::unbounded_channel();
+        let (tx, rx) = mpsc::unbounded_channel();
 
         // Add an entry for this `Peer` in the shared state map.
         state.lock().await.peers.insert(addr, tx);
 
-        let rx = Box::pin(async_stream::stream! {
-            while let Some(item) = rx.recv().await {
-                yield item;
-            }
-        });
-
         Ok(Peer { lines, rx })
     }
 }
 
-#[derive(Debug)]
-enum Message {
-    /// A message that should be broadcasted to others.
-    Broadcast(String),
-
-    /// A message that should be received by a client
-    Received(String),
-}
-
-// Peer implements `Stream` in a way that polls both the `Rx`, and `Framed` types.
-// A message is produced whenever an event is ready until the `Framed` stream returns `None`.
-impl Stream for Peer {
-    type Item = Result<Message, LinesCodecError>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        // First poll the `UnboundedReceiver`.
-
-        if let Poll::Ready(Some(v)) = Pin::new(&mut self.rx).poll_next(cx) {
-            return Poll::Ready(Some(Ok(Message::Received(v))));
-        }
-
-        // Secondly poll the `Framed` stream.
-        let result: Option<_> = futures::ready!(Pin::new(&mut self.lines).poll_next(cx));
-
-        Poll::Ready(match result {
-            // We've received a message we should broadcast to others.
-            Some(Ok(message)) => Some(Ok(Message::Broadcast(message))),
-
-            // An error occurred.
-            Some(Err(e)) => Some(Err(e)),
-
-            // The stream has been exhausted.
-            None => None,
-        })
-    }
-}
-
 /// Process an individual chat client
 async fn process(
     state: Arc<Mutex<Shared>>,
@@ -241,28 +199,32 @@ async fn process(
     }
 
     // Process incoming messages until our stream is exhausted by a disconnect.
-    while let Some(result) = peer.next().await {
-        match result {
-            // A message was received from the current user, we should
-            // broadcast this message to the other users.
-            Ok(Message::Broadcast(msg)) => {
-                let mut state = state.lock().await;
-                let msg = format!("{}: {}", username, msg);
-
-                state.broadcast(addr, &msg).await;
-            }
-            // A message was received from a peer. Send it to the
-            // current user.
-            Ok(Message::Received(msg)) => {
+    loop {
+        tokio::select! {
+            // A message was received from a peer. Send it to the current user.
+            Some(msg) = peer.rx.recv() => {
                 peer.lines.send(&msg).await?;
             }
-            Err(e) => {
-                tracing::error!(
-                    "an error occurred while processing messages for {}; error = {:?}",
-                    username,
-                    e
-                );
-            }
+            result = peer.lines.next() => match result {
+                // A message was received from the current user, we should
+                // broadcast this message to the other users.
+                Some(Ok(msg)) => {
+                    let mut state = state.lock().await;
+                    let msg = format!("{}: {}", username, msg);
+
+                    state.broadcast(addr, &msg).await;
+                }
+                // An error occurred.
+                Some(Err(e)) => {
+                    tracing::error!(
+                        "an error occurred while processing messages for {}; error = {:?}",
+                        username,
+                        e
+                    );
+                }
+                // The stream has been exhausted.
+                None => break,
+            },
         }
     }
 

examples/connect.rs

@@ -92,7 +92,7 @@ mod tcp {
 
 mod udp {
     use bytes::Bytes;
-    use futures::{future, Sink, SinkExt, Stream, StreamExt};
+    use futures::{Sink, SinkExt, Stream, StreamExt};
     use std::error::Error;
     use std::io;
     use std::net::SocketAddr;
@@ -114,7 +114,7 @@ mod udp {
         let socket = UdpSocket::bind(&bind_addr).await?;
         socket.connect(addr).await?;
 
-        future::try_join(send(stdin, &socket), recv(stdout, &socket)).await?;
+        tokio::try_join!(send(stdin, &socket), recv(stdout, &socket))?;
 
         Ok(())
     }

examples/echo.rs

@@ -55,7 +55,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         // which will allow all of our clients to be processed concurrently.
 
         tokio::spawn(async move {
-            let mut buf = [0; 1024];
+            let mut buf = vec![0; 1024];
 
             // In a loop, read data from the socket and write the data back.
             loop {

examples/named-pipe-multi-client.rs

@@ -0,0 +1,98 @@
+use std::io;
+
+#[cfg(windows)]
+async fn windows_main() -> io::Result<()> {
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions};
+    use tokio::time;
+    use winapi::shared::winerror;
+
+    const PIPE_NAME: &str = r"\\.\pipe\named-pipe-multi-client";
+    const N: usize = 10;
+
+    // The first server needs to be constructed early so that clients can
+    // be correctly connected. Otherwise a waiting client will error.
+    //
+    // Here we also make use of `first_pipe_instance`, which will ensure
+    // that there are no other servers up and running already.
+    let mut server = ServerOptions::new()
+        .first_pipe_instance(true)
+        .create(PIPE_NAME)?;
+
+    let server = tokio::spawn(async move {
+        // Artificial workload.
+        time::sleep(Duration::from_secs(1)).await;
+
+        for _ in 0..N {
+            // Wait for client to connect.
+            server.connect().await?;
+            let mut inner = server;
+
+            // Construct the next server to be connected before sending the one
+            // we already have of onto a task. This ensures that the server
+            // isn't closed (after it's done in the task) before a new one is
+            // available. Otherwise the client might error with
+            // `io::ErrorKind::NotFound`.
+            server = ServerOptions::new().create(PIPE_NAME)?;
+
+            let _ = tokio::spawn(async move {
+                let mut buf = vec![0u8; 4];
+                inner.read_exact(&mut buf).await?;
+                inner.write_all(b"pong").await?;
+                Ok::<_, io::Error>(())
+            });
+        }
+
+        Ok::<_, io::Error>(())
+    });
+
+    let mut clients = Vec::new();
+
+    for _ in 0..N {
+        clients.push(tokio::spawn(async move {
+            // This showcases a generic connect loop.
+            //
+            // We immediately try to create a client, if it's not found or
+            // the pipe is busy we use the specialized wait function on the
+            // client builder.
+            let mut client = loop {
+                match ClientOptions::new().open(PIPE_NAME) {
+                    Ok(client) => break client,
+                    Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+                    Err(e) => return Err(e),
+                }
+
+                time::sleep(Duration::from_millis(5)).await;
+            };
+
+            let mut buf = [0u8; 4];
+            client.write_all(b"ping").await?;
+            client.read_exact(&mut buf).await?;
+            Ok::<_, io::Error>(buf)
+        }));
+    }
+
+    for client in clients {
+        let result = client.await?;
+        assert_eq!(&result?[..], b"pong");
+    }
+
+    server.await??;
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> io::Result<()> {
+    #[cfg(windows)]
+    {
+        windows_main().await?;
+    }
+
+    #[cfg(not(windows))]
+    {
+        println!("Named pipes are only supported on Windows!");
+    }
+
+    Ok(())
+}

examples/named-pipe.rs

@@ -0,0 +1,60 @@
+use std::io;
+
+#[cfg(windows)]
+async fn windows_main() -> io::Result<()> {
+    use tokio::io::AsyncWriteExt;
+    use tokio::io::{AsyncBufReadExt, BufReader};
+    use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions};
+
+    const PIPE_NAME: &str = r"\\.\pipe\named-pipe-single-client";
+
+    let server = ServerOptions::new().create(PIPE_NAME)?;
+
+    let server = tokio::spawn(async move {
+        // Note: we wait for a client to connect.
+        server.connect().await?;
+
+        let mut server = BufReader::new(server);
+
+        let mut buf = String::new();
+        server.read_line(&mut buf).await?;
+        server.write_all(b"pong\n").await?;
+        Ok::<_, io::Error>(buf)
+    });
+
+    let client = tokio::spawn(async move {
+        // There's no need to use a connect loop here, since we know that the
+        // server is already up - `open` was called before spawning any of the
+        // tasks.
+        let client = ClientOptions::new().open(PIPE_NAME)?;
+
+        let mut client = BufReader::new(client);
+
+        let mut buf = String::new();
+        client.write_all(b"ping\n").await?;
+        client.read_line(&mut buf).await?;
+        Ok::<_, io::Error>(buf)
+    });
+
+    let (server, client) = tokio::try_join!(server, client)?;
+
+    assert_eq!(server?, "ping\n");
+    assert_eq!(client?, "pong\n");
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> io::Result<()> {
+    #[cfg(windows)]
+    {
+        windows_main().await?;
+    }
+
+    #[cfg(not(windows))]
+    {
+        println!("Named pipes are only supported on Windows!");
+    }
+
+    Ok(())
+}

examples/proxy.rs

@@ -26,7 +26,6 @@ use tokio::io;
 use tokio::io::AsyncWriteExt;
 use tokio::net::{TcpListener, TcpStream};
 
-use futures::future::try_join;
 use futures::FutureExt;
 use std::env;
 use std::error::Error;
@@ -74,7 +73,7 @@ async fn transfer(mut inbound: TcpStream, proxy_addr: String) -> Result<(), Box<
         wi.shutdown().await
     };
 
-    try_join(client_to_server, server_to_client).await?;
+    tokio::try_join!(client_to_server, server_to_client)?;
 
     Ok(())
 }

examples/udp-codec.rs

@@ -45,7 +45,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     let b = pong(&mut b);
 
     // Run both futures simultaneously of `a` and `b` sending messages back and forth.
-    match futures::future::try_join(a, b).await {
+    match tokio::try_join!(a, b) {
         Err(e) => println!("an error occurred; error = {:?}", e),
         _ => println!("done!"),
     }

tests-build/tests/fail/macros_dead_code.rs

@@ -0,0 +1,8 @@
+#![deny(dead_code)]
+
+use tests_build::tokio;
+
+#[tokio::main]
+async fn f() {}
+
+fn main() {}

tests-build/tests/fail/macros_dead_code.stderr

@@ -0,0 +1,11 @@
+error: function is never used: `f`
+ --> $DIR/macros_dead_code.rs:6:10
+  |
+6 | async fn f() {}
+  |          ^
+  |
+note: the lint level is defined here
+ --> $DIR/macros_dead_code.rs:1:9
+  |
+1 | #![deny(dead_code)]
+  |         ^^^^^^^^^

tests-build/tests/fail/macros_invalid_input.rs

@@ -12,12 +12,27 @@ async fn main_attr_has_path_args() {}
 #[tokio::test]
 fn test_is_not_async() {}
 
-#[tokio::test]
-async fn test_fn_has_args(_x: u8) {}
-
 #[tokio::test(foo)]
 async fn test_attr_has_args() {}
 
+#[tokio::test(foo = 123)]
+async fn test_unexpected_attr() {}
+
+#[tokio::test(flavor = 123)]
+async fn test_flavor_not_string() {}
+
+#[tokio::test(flavor = "foo")]
+async fn test_unknown_flavor() {}
+
+#[tokio::test(flavor = "multi_thread", start_paused = false)]
+async fn test_multi_thread_with_start_paused() {}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = "foo")]
+async fn test_worker_threads_not_int() {}
+
+#[tokio::test(flavor = "current_thread", worker_threads = 4)]
+async fn test_worker_threads_and_current_thread() {}
+
 #[tokio::test]
 #[test]
 async fn test_has_second_test_attr() {}

tests-build/tests/fail/macros_invalid_input.stderr

@@ -1,10 +1,10 @@
-error: the async keyword is missing from the function declaration
+error: the `async` keyword is missing from the function declaration
  --> $DIR/macros_invalid_input.rs:4:1
   |
 4 | fn main_is_not_async() {}
   | ^^
 
-error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
  --> $DIR/macros_invalid_input.rs:6:15
   |
 6 | #[tokio::main(foo)]
@@ -16,26 +16,56 @@ error: Must have specified ident
 9 | #[tokio::main(threadpool::bar)]
   |               ^^^^^^^^^^^^^^^
 
-error: the async keyword is missing from the function declaration
+error: the `async` keyword is missing from the function declaration
   --> $DIR/macros_invalid_input.rs:13:1
    |
 13 | fn test_is_not_async() {}
    | ^^
 
-error: the test function cannot accept arguments
-  --> $DIR/macros_invalid_input.rs:16:27
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
+  --> $DIR/macros_invalid_input.rs:15:15
    |
-16 | async fn test_fn_has_args(_x: u8) {}
-   |                           ^^^^^^
-
-error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`
-  --> $DIR/macros_invalid_input.rs:18:15
-   |
-18 | #[tokio::test(foo)]
+15 | #[tokio::test(foo)]
    |               ^^^
 
-error: second test attribute is supplied
-  --> $DIR/macros_invalid_input.rs:22:1
+error: Unknown attribute foo is specified; expected one of: `flavor`, `worker_threads`, `start_paused`
+  --> $DIR/macros_invalid_input.rs:18:15
    |
-22 | #[test]
+18 | #[tokio::test(foo = 123)]
+   |               ^^^^^^^^^
+
+error: Failed to parse value of `flavor` as string.
+  --> $DIR/macros_invalid_input.rs:21:24
+   |
+21 | #[tokio::test(flavor = 123)]
+   |                        ^^^
+
+error: No such runtime flavor `foo`. The runtime flavors are `current_thread` and `multi_thread`.
+  --> $DIR/macros_invalid_input.rs:24:24
+   |
+24 | #[tokio::test(flavor = "foo")]
+   |                        ^^^^^
+
+error: The `start_paused` option requires the `current_thread` runtime flavor. Use `#[tokio::test(flavor = "current_thread")]`
+  --> $DIR/macros_invalid_input.rs:27:55
+   |
+27 | #[tokio::test(flavor = "multi_thread", start_paused = false)]
+   |                                                       ^^^^^
+
+error: Failed to parse value of `worker_threads` as integer.
+  --> $DIR/macros_invalid_input.rs:30:57
+   |
+30 | #[tokio::test(flavor = "multi_thread", worker_threads = "foo")]
+   |                                                         ^^^^^
+
+error: The `worker_threads` option requires the `multi_thread` runtime flavor. Use `#[tokio::test(flavor = "multi_thread")]`
+  --> $DIR/macros_invalid_input.rs:33:59
+   |
+33 | #[tokio::test(flavor = "current_thread", worker_threads = 4)]
+   |                                                           ^
+
+error: second test attribute is supplied
+  --> $DIR/macros_invalid_input.rs:37:1
+   |
+37 | #[test]
    | ^^^^^^^

tests-build/tests/fail/macros_type_mismatch.rs

@@ -0,0 +1,32 @@
+use tests_build::tokio;
+
+#[tokio::main]
+async fn missing_semicolon_or_return_type() {
+    Ok(())
+}
+
+#[tokio::main]
+async fn missing_return_type() {
+    /* TODO(taiki-e): one of help messages still wrong
+    help: consider using a semicolon here
+       |
+    16 |     return Ok(());;
+       |
+    */
+    return Ok(());
+}
+
+#[tokio::main]
+async fn extra_semicolon() -> Result<(), ()> {
+    /* TODO(taiki-e): help message still wrong
+    help: try using a variant of the expected enum
+       |
+    29 |     Ok(Ok(());)
+       |
+    29 |     Err(Ok(());)
+       |
+    */
+    Ok(());
+}
+
+fn main() {}

tests-build/tests/fail/macros_type_mismatch.stderr

@@ -0,0 +1,51 @@
+error[E0308]: mismatched types
+ --> $DIR/macros_type_mismatch.rs:5:5
+  |
+5 |     Ok(())
+  |     ^^^^^^ expected `()`, found enum `Result`
+  |
+  = note: expected unit type `()`
+                  found enum `Result<(), _>`
+help: consider using a semicolon here
+  |
+5 |     Ok(());
+  |           ^
+help: try adding a return type
+  |
+4 | async fn missing_semicolon_or_return_type() -> Result<(), _> {
+  |                                             ^^^^^^^^^^^^^^^^
+
+error[E0308]: mismatched types
+  --> $DIR/macros_type_mismatch.rs:16:5
+   |
+16 |     return Ok(());
+   |     ^^^^^^^^^^^^^^ expected `()`, found enum `Result`
+   |
+   = note: expected unit type `()`
+                   found enum `Result<(), _>`
+help: consider using a semicolon here
+   |
+16 |     return Ok(());;
+   |                   ^
+help: try adding a return type
+   |
+9  | async fn missing_return_type() -> Result<(), _> {
+   |                                ^^^^^^^^^^^^^^^^
+
+error[E0308]: mismatched types
+  --> $DIR/macros_type_mismatch.rs:29:5
+   |
+20 | async fn extra_semicolon() -> Result<(), ()> {
+   |                               -------------- expected `Result<(), ()>` because of return type
+...
+29 |     Ok(());
+   |     ^^^^^^^ expected enum `Result`, found `()`
+   |
+   = note:   expected enum `Result<(), ()>`
+           found unit type `()`
+help: try using a variant of the expected enum
+   |
+29 |     Ok(Ok(());)
+   |
+29 |     Err(Ok(());)
+   |

tests-build/tests/macros.rs

@@ -2,9 +2,18 @@
 fn compile_fail_full() {
     let t = trybuild::TestCases::new();
 
+    #[cfg(feature = "full")]
+    t.pass("tests/pass/forward_args_and_output.rs");
+
     #[cfg(feature = "full")]
     t.compile_fail("tests/fail/macros_invalid_input.rs");
 
+    #[cfg(feature = "full")]
+    t.compile_fail("tests/fail/macros_dead_code.rs");
+
+    #[cfg(feature = "full")]
+    t.compile_fail("tests/fail/macros_type_mismatch.rs");
+
     #[cfg(all(feature = "rt", not(feature = "full")))]
     t.compile_fail("tests/fail/macros_core_no_default.rs");
 

tests-build/tests/pass/forward_args_and_output.rs

@@ -0,0 +1,13 @@
+use tests_build::tokio;
+
+fn main() {}
+
+// arguments and output type is forwarded so other macros can access them
+
+#[tokio::test]
+async fn test_fn_has_args(_x: u8) {}
+
+#[tokio::test]
+async fn test_has_output() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(())
+}

tests-integration/Cargo.toml

@@ -12,9 +12,15 @@ name = "test-cat"
 name = "test-mem"
 required-features = ["rt-net"]
 
+[[bin]]
+name = "test-process-signal"
+required-features = ["rt-process-signal"]
+
 [features]
 # For mem check
 rt-net = ["tokio/rt", "tokio/rt-multi-thread", "tokio/net"]
+# For test-process-signal
+rt-process-signal = ["rt", "tokio/process", "tokio/signal"]
 
 full = [
     "macros",

tests-integration/src/bin/test-process-signal.rs

@@ -0,0 +1,11 @@
+// https://github.com/tokio-rs/tokio/issues/3550
+fn main() {
+    for _ in 0..1000 {
+        let rt = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        drop(rt);
+    }
+}

tests-integration/tests/process_stdio.rs

@@ -1,26 +1,19 @@
 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 
-use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
+use tokio::join;
 use tokio::process::{Child, Command};
 use tokio_test::assert_ok;
 
 use futures::future::{self, FutureExt};
+use std::convert::TryInto;
 use std::env;
 use std::io;
 use std::process::{ExitStatus, Stdio};
 
 fn cat() -> Command {
-    let mut me = env::current_exe().unwrap();
-    me.pop();
-
-    if me.ends_with("deps") {
-        me.pop();
-    }
-
-    me.push("test-cat");
-
-    let mut cmd = Command::new(me);
+    let mut cmd = Command::new(env!("CARGO_BIN_EXE_test-cat"));
     cmd.stdin(Stdio::piped()).stdout(Stdio::piped());
     cmd
 }
@@ -148,3 +141,52 @@ async fn try_wait() {
     // Can't get id after process has exited
     assert_eq!(child.id(), None);
 }
+
+#[tokio::test]
+async fn pipe_from_one_command_to_another() {
+    let mut first = cat().spawn().expect("first cmd");
+    let mut third = cat().spawn().expect("third cmd");
+
+    // Convert ChildStdout to Stdio
+    let second_stdin: Stdio = first
+        .stdout
+        .take()
+        .expect("first.stdout")
+        .try_into()
+        .expect("first.stdout into Stdio");
+
+    // Convert ChildStdin to Stdio
+    let second_stdout: Stdio = third
+        .stdin
+        .take()
+        .expect("third.stdin")
+        .try_into()
+        .expect("third.stdin into Stdio");
+
+    let mut second = cat()
+        .stdin(second_stdin)
+        .stdout(second_stdout)
+        .spawn()
+        .expect("first cmd");
+
+    let msg = "hello world! please pipe this message through";
+
+    let mut stdin = first.stdin.take().expect("first.stdin");
+    let write = async move { stdin.write_all(msg.as_bytes()).await };
+
+    let mut stdout = third.stdout.take().expect("third.stdout");
+    let read = async move {
+        let mut data = String::new();
+        stdout.read_to_string(&mut data).await.map(|_| data)
+    };
+
+    let (read, write, first_status, second_status, third_status) =
+        join!(read, write, first.wait(), second.wait(), third.wait());
+
+    assert_eq!(msg, read.expect("read result"));
+    write.expect("write result");
+
+    assert!(first_status.expect("first status").success());
+    assert!(second_status.expect("second status").success());
+    assert!(third_status.expect("third status").success());
+}

tokio-macros/CHANGELOG.md

@@ -1,3 +1,17 @@
+# 1.2.0 (May 14, 2021)
+
+- macros: forward input arguments in `#[tokio::test]` ([#3691])
+- macros: improve diagnostics on type mismatch ([#3766])
+- macros: various error message improvements ([#3677])
+
+[#3677]: https://github.com/tokio-rs/tokio/pull/3677
+[#3691]: https://github.com/tokio-rs/tokio/pull/3691
+[#3766]: https://github.com/tokio-rs/tokio/pull/3766
+
+# 1.1.0 (February 5, 2021)
+
+- add `start_paused` option to macros ([#3492])
+
 # 1.0.0 (December 23, 2020)
 
 - track `tokio` 1.0 release.
@@ -55,3 +69,4 @@
 [#2177]: https://github.com/tokio-rs/tokio/pull/2177
 [#2225]: https://github.com/tokio-rs/tokio/pull/2225
 [#3038]: https://github.com/tokio-rs/tokio/pull/3038
+[#3492]: https://github.com/tokio-rs/tokio/pull/3492

tokio-macros/Cargo.toml

@@ -2,18 +2,17 @@
 name = "tokio-macros"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update html_root_url.
 # - Update doc url
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-macros-1.0.x" git tag.
-version = "1.0.0"
+version = "1.2.0"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-macros/1.0.0/tokio_macros"
+documentation = "https://docs.rs/tokio-macros/1.2.0/tokio_macros"
 description = """
 Tokio's proc macros.
 """
@@ -27,7 +26,7 @@ proc-macro = true
 [dependencies]
 proc-macro2 = "1.0.7"
 quote = "1"
-syn = { version = "1.0.3", features = ["full"] }
+syn = { version = "1.0.56", features = ["full"] }
 
 [dev-dependencies]
 tokio = { version = "1.0.0", path = "../tokio", features = ["full"] }

tokio-macros/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Tokio Contributors
+Copyright (c) 2021 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-macros/src/entry.rs

@@ -1,7 +1,6 @@
 use proc_macro::TokenStream;
 use proc_macro2::Span;
-use quote::quote;
-use syn::spanned::Spanned;
+use quote::{quote, quote_spanned, ToTokens};
 
 #[derive(Clone, Copy, PartialEq)]
 enum RuntimeFlavor {
@@ -25,6 +24,7 @@ impl RuntimeFlavor {
 struct FinalConfig {
     flavor: RuntimeFlavor,
     worker_threads: Option<usize>,
+    start_paused: Option<bool>,
 }
 
 struct Configuration {
@@ -32,6 +32,8 @@ struct Configuration {
     default_flavor: RuntimeFlavor,
     flavor: Option<RuntimeFlavor>,
     worker_threads: Option<(usize, Span)>,
+    start_paused: Option<(bool, Span)>,
+    is_test: bool,
 }
 
 impl Configuration {
@@ -44,6 +46,8 @@ impl Configuration {
             },
             flavor: None,
             worker_threads: None,
+            start_paused: None,
+            is_test,
         }
     }
 
@@ -79,31 +83,67 @@ impl Configuration {
         Ok(())
     }
 
+    fn set_start_paused(&mut self, start_paused: syn::Lit, span: Span) -> Result<(), syn::Error> {
+        if self.start_paused.is_some() {
+            return Err(syn::Error::new(span, "`start_paused` set multiple times."));
+        }
+
+        let start_paused = parse_bool(start_paused, span, "start_paused")?;
+        self.start_paused = Some((start_paused, span));
+        Ok(())
+    }
+
+    fn macro_name(&self) -> &'static str {
+        if self.is_test {
+            "tokio::test"
+        } else {
+            "tokio::main"
+        }
+    }
+
     fn build(&self) -> Result<FinalConfig, syn::Error> {
         let flavor = self.flavor.unwrap_or(self.default_flavor);
         use RuntimeFlavor::*;
-        match (flavor, self.worker_threads) {
-            (CurrentThread, Some((_, worker_threads_span))) => Err(syn::Error::new(
-                worker_threads_span,
-                "The `worker_threads` option requires the `multi_thread` runtime flavor.",
-            )),
-            (CurrentThread, None) => Ok(FinalConfig {
-                flavor,
-                worker_threads: None,
-            }),
-            (Threaded, worker_threads) if self.rt_multi_thread_available => Ok(FinalConfig {
-                flavor,
-                worker_threads: worker_threads.map(|(val, _span)| val),
-            }),
+
+        let worker_threads = match (flavor, self.worker_threads) {
+            (CurrentThread, Some((_, worker_threads_span))) => {
+                let msg = format!(
+                    "The `worker_threads` option requires the `multi_thread` runtime flavor. Use `#[{}(flavor = \"multi_thread\")]`",
+                    self.macro_name(),
+                );
+                return Err(syn::Error::new(worker_threads_span, msg));
+            }
+            (CurrentThread, None) => None,
+            (Threaded, worker_threads) if self.rt_multi_thread_available => {
+                worker_threads.map(|(val, _span)| val)
+            }
             (Threaded, _) => {
                 let msg = if self.flavor.is_none() {
                     "The default runtime flavor is `multi_thread`, but the `rt-multi-thread` feature is disabled."
                 } else {
                     "The runtime flavor `multi_thread` requires the `rt-multi-thread` feature."
                 };
-                Err(syn::Error::new(Span::call_site(), msg))
+                return Err(syn::Error::new(Span::call_site(), msg));
             }
-        }
+        };
+
+        let start_paused = match (flavor, self.start_paused) {
+            (Threaded, Some((_, start_paused_span))) => {
+                let msg = format!(
+                    "The `start_paused` option requires the `current_thread` runtime flavor. Use `#[{}(flavor = \"current_thread\")]`",
+                    self.macro_name(),
+                );
+                return Err(syn::Error::new(start_paused_span, msg));
+            }
+            (CurrentThread, Some((start_paused, _))) => Some(start_paused),
+            (_, None) => None,
+        };
+
+        Ok(FinalConfig {
+            flavor,
+            worker_threads,
+            start_paused,
+        })
     }
 }
 
@@ -113,12 +153,12 @@ fn parse_int(int: syn::Lit, span: Span, field: &str) -> Result<usize, syn::Error
             Ok(value) => Ok(value),
             Err(e) => Err(syn::Error::new(
                 span,
-                format!("Failed to parse {} as integer: {}", field, e),
+                format!("Failed to parse value of `{}` as integer: {}", field, e),
             )),
         },
         _ => Err(syn::Error::new(
             span,
-            format!("Failed to parse {} as integer.", field),
+            format!("Failed to parse value of `{}` as integer.", field),
         )),
     }
 }
@@ -129,7 +169,17 @@ fn parse_string(int: syn::Lit, span: Span, field: &str) -> Result<String, syn::E
         syn::Lit::Verbatim(s) => Ok(s.to_string()),
         _ => Err(syn::Error::new(
             span,
-            format!("Failed to parse {} as string.", field),
+            format!("Failed to parse value of `{}` as string.", field),
+        )),
+    }
+}
+
+fn parse_bool(bool: syn::Lit, span: Span, field: &str) -> Result<bool, syn::Error> {
+    match bool {
+        syn::Lit::Bool(b) => Ok(b.value),
+        _ => Err(syn::Error::new(
+            span,
+            format!("Failed to parse value of `{}` as bool.", field),
         )),
     }
 }
@@ -140,24 +190,13 @@ fn parse_knobs(
     is_test: bool,
     rt_multi_thread: bool,
 ) -> Result<TokenStream, syn::Error> {
-    let sig = &mut input.sig;
-    let body = &input.block;
-    let attrs = &input.attrs;
-    let vis = input.vis;
-
-    if sig.asyncness.is_none() {
-        let msg = "the async keyword is missing from the function declaration";
-        return Err(syn::Error::new_spanned(sig.fn_token, msg));
+    if input.sig.asyncness.take().is_none() {
+        let msg = "the `async` keyword is missing from the function declaration";
+        return Err(syn::Error::new_spanned(input.sig.fn_token, msg));
     }
 
-    sig.asyncness = None;
-
-    let macro_name = if is_test {
-        "tokio::test"
-    } else {
-        "tokio::main"
-    };
     let mut config = Configuration::new(is_test, rt_multi_thread);
+    let macro_name = config.macro_name();
 
     for arg in args {
         match arg {
@@ -169,17 +208,32 @@ fn parse_knobs(
                 }
                 match ident.unwrap().to_string().to_lowercase().as_str() {
                     "worker_threads" => {
-                        config.set_worker_threads(namevalue.lit.clone(), namevalue.span())?;
+                        config.set_worker_threads(
+                            namevalue.lit.clone(),
+                            syn::spanned::Spanned::span(&namevalue.lit),
+                        )?;
                     }
                     "flavor" => {
-                        config.set_flavor(namevalue.lit.clone(), namevalue.span())?;
+                        config.set_flavor(
+                            namevalue.lit.clone(),
+                            syn::spanned::Spanned::span(&namevalue.lit),
+                        )?;
+                    }
+                    "start_paused" => {
+                        config.set_start_paused(
+                            namevalue.lit.clone(),
+                            syn::spanned::Spanned::span(&namevalue.lit),
+                        )?;
                     }
                     "core_threads" => {
                         let msg = "Attribute `core_threads` is renamed to `worker_threads`";
                         return Err(syn::Error::new_spanned(namevalue, msg));
                     }
                     name => {
-                        let msg = format!("Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`", name);
+                        let msg = format!(
+                            "Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`",
+                            name,
+                        );
                         return Err(syn::Error::new_spanned(namevalue, msg));
                     }
                 }
@@ -204,11 +258,11 @@ fn parse_knobs(
                             macro_name
                         )
                     }
-                    "flavor" | "worker_threads" => {
+                    "flavor" | "worker_threads" | "start_paused" => {
                         format!("The `{}` attribute requires an argument.", name)
                     }
                     name => {
-                        format!("Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`", name)
+                        format!("Unknown attribute {} is specified; expected one of: `flavor`, `worker_threads`, `start_paused`", name)
                     }
                 };
                 return Err(syn::Error::new_spanned(path, msg));
@@ -224,38 +278,64 @@ fn parse_knobs(
 
     let config = config.build()?;
 
+    // If type mismatch occurs, the current rustc points to the last statement.
+    let (last_stmt_start_span, last_stmt_end_span) = {
+        let mut last_stmt = input
+            .block
+            .stmts
+            .last()
+            .map(ToTokens::into_token_stream)
+            .unwrap_or_default()
+            .into_iter();
+        // `Span` on stable Rust has a limitation that only points to the first
+        // token, not the whole tokens. We can work around this limitation by
+        // using the first/last span of the tokens like
+        // `syn::Error::new_spanned` does.
+        let start = last_stmt.next().map_or_else(Span::call_site, |t| t.span());
+        let end = last_stmt.last().map_or(start, |t| t.span());
+        (start, end)
+    };
+
     let mut rt = match config.flavor {
-        RuntimeFlavor::CurrentThread => quote! {
+        RuntimeFlavor::CurrentThread => quote_spanned! {last_stmt_start_span=>
             tokio::runtime::Builder::new_current_thread()
         },
-        RuntimeFlavor::Threaded => quote! {
+        RuntimeFlavor::Threaded => quote_spanned! {last_stmt_start_span=>
             tokio::runtime::Builder::new_multi_thread()
         },
     };
     if let Some(v) = config.worker_threads {
         rt = quote! { #rt.worker_threads(#v) };
     }
+    if let Some(v) = config.start_paused {
+        rt = quote! { #rt.start_paused(#v) };
+    }
 
-    let header = {
-        if is_test {
-            quote! {
-                #[::core::prelude::v1::test]
-            }
-        } else {
-            quote! {}
+    let header = if is_test {
+        quote! {
+            #[::core::prelude::v1::test]
         }
+    } else {
+        quote! {}
     };
 
-    let result = quote! {
-        #header
-        #(#attrs)*
-        #vis #sig {
+    let body = &input.block;
+    let brace_token = input.block.brace_token;
+    input.block = syn::parse2(quote_spanned! {last_stmt_end_span=>
+        {
             #rt
                 .enable_all()
                 .build()
                 .unwrap()
-                .block_on(async { #body })
+                .block_on(async #body)
         }
+    })
+    .unwrap();
+    input.block.brace_token = brace_token;
+
+    let result = quote! {
+        #header
+        #input
     };
 
     Ok(result.into())
@@ -289,12 +369,5 @@ pub(crate) fn test(args: TokenStream, item: TokenStream, rt_multi_thread: bool)
         }
     }
 
-    if !input.sig.inputs.is_empty() {
-        let msg = "the test function cannot accept arguments";
-        return syn::Error::new_spanned(&input.sig.inputs, msg)
-            .to_compile_error()
-            .into();
-    }
-
     parse_knobs(input, args, true, rt_multi_thread).unwrap_or_else(|e| e.to_compile_error().into())
 }

tokio-macros/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc(html_root_url = "https://docs.rs/tokio-macros/1.0.0")]
 #![allow(clippy::needless_doctest_main)]
 #![warn(
     missing_debug_implementations,
@@ -144,6 +143,32 @@ use proc_macro::TokenStream;
 /// }
 /// ```
 ///
+/// ### Configure the runtime to start with time paused
+///
+/// ```rust
+/// #[tokio::main(flavor = "current_thread", start_paused = true)]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new_current_thread()
+///         .enable_all()
+///         .start_paused(true)
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// Note that `start_paused` requires the `test-util` feature to be enabled.
+///
 /// ### NOTE:
 ///
 /// If you rename the Tokio crate in your dependencies this macro will not work.
@@ -225,6 +250,17 @@ pub fn main_rt(args: TokenStream, item: TokenStream) -> TokenStream {
 /// }
 /// ```
 ///
+/// ### Configure the runtime to start with time paused
+///
+/// ```no_run
+/// #[tokio::test(start_paused = true)]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// Note that `start_paused` requires the `test-util` feature to be enabled.
+///
 /// ### NOTE:
 ///
 /// If you rename the Tokio crate in your dependencies this macro will not work.

tokio-stream/CHANGELOG.md

@@ -1,3 +1,69 @@
+# 0.1.6 (May 14, 2021)
+
+### Added
+
+- stream: implement `Error` and `Display` for `BroadcastStreamRecvError` ([#3745])
+
+### Fixed
+
+- stream: avoid yielding in `AllFuture` and `AnyFuture` ([#3625])
+
+[#3745]: https://github.com/tokio-rs/tokio/pull/3745
+[#3625]: https://github.com/tokio-rs/tokio/pull/3625
+
+# 0.1.5 (March 20, 2021)
+
+### Fixed
+
+- stream: documentation note for throttle `Unpin` ([#3600])
+
+[#3600]: https://github.com/tokio-rs/tokio/pull/3600
+
+# 0.1.4 (March 9, 2021)
+
+Added
+
+- signal: add `Signal` wrapper ([#3510])
+
+Fixed
+
+- stream: remove duplicate `doc_cfg` declaration ([#3561])
+- sync: yield initial value in `WatchStream` ([#3576])
+
+[#3510]: https://github.com/tokio-rs/tokio/pull/3510
+[#3561]: https://github.com/tokio-rs/tokio/pull/3561
+[#3576]: https://github.com/tokio-rs/tokio/pull/3576
+
+# 0.1.3 (February 5, 2021)
+
+Added
+
+ - sync: add wrapper for broadcast and watch ([#3384], [#3504])
+
+[#3384]: https://github.com/tokio-rs/tokio/pull/3384
+[#3504]: https://github.com/tokio-rs/tokio/pull/3504
+
+# 0.1.2 (January 12, 2021)
+
+Fixed
+
+ - docs: fix some wrappers missing in documentation ([#3378])
+
+[#3378]: https://github.com/tokio-rs/tokio/pull/3378
+
+# 0.1.1 (January 4, 2021)
+
+Added
+
+ - add `Stream` wrappers ([#3343])
+
+Fixed
+
+ - move `async-stream` to `dev-dependencies` ([#3366])
+
+[#3366]: https://github.com/tokio-rs/tokio/pull/3366
+[#3343]: https://github.com/tokio-rs/tokio/pull/3343
+
 # 0.1.0 (December 23, 2020)
 
-- Initial release
+ - Initial release

tokio-stream/Cargo.toml

@@ -2,18 +2,17 @@
 name = "tokio-stream"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update html_root_url.
 # - Update doc url
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-stream-0.1.x" git tag.
-version = "0.1.0"
+version = "0.1.6"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-stream/0.1.0/tokio_stream"
+documentation = "https://docs.rs/tokio-stream/0.1.6/tokio_stream"
 description = """
 Utilities to work with `Stream` and `tokio`.
 """
@@ -22,16 +21,26 @@ categories = ["asynchronous"]
 [features]
 default = ["time"]
 time = ["tokio/time"]
+net = ["tokio/net"]
+io-util = ["tokio/io-util"]
+fs = ["tokio/fs"]
+sync = ["tokio/sync", "tokio-util"]
+signal = ["tokio/signal"]
 
 [dependencies]
 futures-core = { version = "0.3.0" }
 pin-project-lite = "0.2.0"
-tokio = { version = "1.0", path = "../tokio", features = ["sync"] }
-async-stream = "0.3"
+tokio = { version = "1.2.0", path = "../tokio", features = ["sync"] }
+tokio-util = { version = "0.6.3", path = "../tokio-util", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.0", path = "../tokio", features = ["full"] }
+tokio = { version = "1.2.0", path = "../tokio", features = ["full", "test-util"] }
+async-stream = "0.3"
 tokio-test = { path = "../tokio-test" }
 futures = { version = "0.3", default-features = false }
 
-proptest = "0.10.0"
+proptest = "1"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]

tokio-stream/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Tokio Contributors
+Copyright (c) 2021 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-stream/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc(html_root_url = "https://docs.rs/tokio-stream/0.1.0")]
 #![allow(
     clippy::cognitive_complexity,
     clippy::large_enum_variant,
@@ -10,18 +9,12 @@
     rust_2018_idioms,
     unreachable_pub
 )]
-#![cfg_attr(docsrs, deny(broken_intra_doc_links))]
-#![doc(test(
-    no_crate_inject,
-    attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
-))]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 #![cfg_attr(docsrs, deny(broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
 ))]
-#![cfg_attr(docsrs, feature(doc_cfg))]
 
 //! Stream utilities for Tokio.
 //!
@@ -81,46 +74,17 @@
 #[macro_use]
 mod macros;
 
-mod all;
-use all::AllFuture;
+pub mod wrappers;
 
-mod any;
-use any::AnyFuture;
-
-mod chain;
-use chain::Chain;
-
-mod collect;
-use collect::Collect;
-pub use collect::FromStream;
+mod stream_ext;
+pub use stream_ext::{collect::FromStream, StreamExt};
 
 mod empty;
 pub use empty::{empty, Empty};
 
-mod filter;
-use filter::Filter;
-
-mod filter_map;
-use filter_map::FilterMap;
-
-mod fold;
-use fold::FoldFuture;
-
-mod fuse;
-use fuse::Fuse;
-
 mod iter;
 pub use iter::{iter, Iter};
 
-mod map;
-use map::Map;
-
-mod merge;
-use merge::Merge;
-
-mod next;
-use next::Next;
-
 mod once;
 pub use once::{once, Once};
 
@@ -130,888 +94,5 @@ pub use pending::{pending, Pending};
 mod stream_map;
 pub use stream_map::StreamMap;
 
-mod skip;
-use skip::Skip;
-
-mod skip_while;
-use skip_while::SkipWhile;
-
-mod try_next;
-use try_next::TryNext;
-
-mod take;
-use take::Take;
-
-mod take_while;
-use take_while::TakeWhile;
-
-cfg_time! {
-    mod timeout;
-    use timeout::Timeout;
-    use tokio::time::Duration;
-    mod throttle;
-    use crate::throttle::{throttle, Throttle};
-}
-
 #[doc(no_inline)]
 pub use futures_core::Stream;
-
-/// An extension trait for the [`Stream`] trait that provides a variety of
-/// convenient combinator functions.
-///
-/// Be aware that the `Stream` trait in Tokio is a re-export of the trait found
-/// in the [futures] crate, however both Tokio and futures provide separate
-/// `StreamExt` utility traits, and some utilities are only available on one of
-/// these traits. Click [here][futures-StreamExt] to see the other `StreamExt`
-/// trait in the futures crate.
-///
-/// If you need utilities from both `StreamExt` traits, you should prefer to
-/// import one of them, and use the other through the fully qualified call
-/// syntax. For example:
-/// ```
-/// // import one of the traits:
-/// use futures::stream::StreamExt;
-/// # #[tokio::main(flavor = "current_thread")]
-/// # async fn main() {
-///
-/// let a = tokio_stream::iter(vec![1, 3, 5]);
-/// let b = tokio_stream::iter(vec![2, 4, 6]);
-///
-/// // use the fully qualified call syntax for the other trait:
-/// let merged = tokio_stream::StreamExt::merge(a, b);
-///
-/// // use normal call notation for futures::stream::StreamExt::collect
-/// let output: Vec<_> = merged.collect().await;
-/// assert_eq!(output, vec![1, 2, 3, 4, 5, 6]);
-/// # }
-/// ```
-///
-/// [`Stream`]: crate::Stream
-/// [futures]: https://docs.rs/futures
-/// [futures-StreamExt]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html
-pub trait StreamExt: Stream {
-    /// Consumes and returns the next value in the stream or `None` if the
-    /// stream is finished.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn next(&mut self) -> Option<Self::Item>;
-    /// ```
-    ///
-    /// Note that because `next` doesn't take ownership over the stream,
-    /// the [`Stream`] type must be [`Unpin`]. If you want to use `next` with a
-    /// [`!Unpin`](Unpin) stream, you'll first have to pin the stream. This can
-    /// be done by boxing the stream using [`Box::pin`] or
-    /// pinning it to the stack using the `pin_mut!` macro from the `pin_utils`
-    /// crate.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let mut stream = stream::iter(1..=3);
-    ///
-    /// assert_eq!(stream.next().await, Some(1));
-    /// assert_eq!(stream.next().await, Some(2));
-    /// assert_eq!(stream.next().await, Some(3));
-    /// assert_eq!(stream.next().await, None);
-    /// # }
-    /// ```
-    fn next(&mut self) -> Next<'_, Self>
-    where
-        Self: Unpin,
-    {
-        Next::new(self)
-    }
-
-    /// Consumes and returns the next item in the stream. If an error is
-    /// encountered before the next item, the error is returned instead.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn try_next(&mut self) -> Result<Option<T>, E>;
-    /// ```
-    ///
-    /// This is similar to the [`next`](StreamExt::next) combinator,
-    /// but returns a [`Result<Option<T>, E>`](Result) rather than
-    /// an [`Option<Result<T, E>>`](Option), making for easy use
-    /// with the [`?`](std::ops::Try) operator.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let mut stream = stream::iter(vec![Ok(1), Ok(2), Err("nope")]);
-    ///
-    /// assert_eq!(stream.try_next().await, Ok(Some(1)));
-    /// assert_eq!(stream.try_next().await, Ok(Some(2)));
-    /// assert_eq!(stream.try_next().await, Err("nope"));
-    /// # }
-    /// ```
-    fn try_next<T, E>(&mut self) -> TryNext<'_, Self>
-    where
-        Self: Stream<Item = Result<T, E>> + Unpin,
-    {
-        TryNext::new(self)
-    }
-
-    /// Maps this stream's items to a different type, returning a new stream of
-    /// the resulting type.
-    ///
-    /// The provided closure is executed over all elements of this stream as
-    /// they are made available. It is executed inline with calls to
-    /// [`poll_next`](Stream::poll_next).
-    ///
-    /// Note that this function consumes the stream passed into it and returns a
-    /// wrapped version of it, similar to the existing `map` methods in the
-    /// standard library.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let stream = stream::iter(1..=3);
-    /// let mut stream = stream.map(|x| x + 3);
-    ///
-    /// assert_eq!(stream.next().await, Some(4));
-    /// assert_eq!(stream.next().await, Some(5));
-    /// assert_eq!(stream.next().await, Some(6));
-    /// # }
-    /// ```
-    fn map<T, F>(self, f: F) -> Map<Self, F>
-    where
-        F: FnMut(Self::Item) -> T,
-        Self: Sized,
-    {
-        Map::new(self, f)
-    }
-
-    /// Combine two streams into one by interleaving the output of both as it
-    /// is produced.
-    ///
-    /// Values are produced from the merged stream in the order they arrive from
-    /// the two source streams. If both source streams provide values
-    /// simultaneously, the merge stream alternates between them. This provides
-    /// some level of fairness. You should not chain calls to `merge`, as this
-    /// will break the fairness of the merging.
-    ///
-    /// The merged stream completes once **both** source streams complete. When
-    /// one source stream completes before the other, the merge stream
-    /// exclusively polls the remaining stream.
-    ///
-    /// For merging multiple streams, consider using [`StreamMap`] instead.
-    ///
-    /// [`StreamMap`]: crate::StreamMap
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use tokio_stream::{StreamExt, Stream};
-    /// use tokio::sync::mpsc;
-    /// use tokio::time;
-    ///
-    /// use std::time::Duration;
-    /// use std::pin::Pin;
-    ///
-    /// # /*
-    /// #[tokio::main]
-    /// # */
-    /// # #[tokio::main(flavor = "current_thread")]
-    /// async fn main() {
-    /// # time::pause();
-    ///     let (tx1, mut rx1) = mpsc::channel::<usize>(10);
-    ///     let (tx2, mut rx2) = mpsc::channel::<usize>(10);
-    ///
-    ///     // Convert the channels to a `Stream`.
-    ///     let rx1 = Box::pin(async_stream::stream! {
-    ///           while let Some(item) = rx1.recv().await {
-    ///               yield item;
-    ///           }
-    ///     }) as Pin<Box<dyn Stream<Item = usize> + Send>>;
-    ///
-    ///     let rx2 = Box::pin(async_stream::stream! {
-    ///           while let Some(item) = rx2.recv().await {
-    ///               yield item;
-    ///           }
-    ///     }) as Pin<Box<dyn Stream<Item = usize> + Send>>;
-    ///
-    ///     let mut rx = rx1.merge(rx2);
-    ///
-    ///     tokio::spawn(async move {
-    ///         // Send some values immediately
-    ///         tx1.send(1).await.unwrap();
-    ///         tx1.send(2).await.unwrap();
-    ///
-    ///         // Let the other task send values
-    ///         time::sleep(Duration::from_millis(20)).await;
-    ///
-    ///         tx1.send(4).await.unwrap();
-    ///     });
-    ///
-    ///     tokio::spawn(async move {
-    ///         // Wait for the first task to send values
-    ///         time::sleep(Duration::from_millis(5)).await;
-    ///
-    ///         tx2.send(3).await.unwrap();
-    ///
-    ///         time::sleep(Duration::from_millis(25)).await;
-    ///
-    ///         // Send the final value
-    ///         tx2.send(5).await.unwrap();
-    ///     });
-    ///
-    ///    assert_eq!(1, rx.next().await.unwrap());
-    ///    assert_eq!(2, rx.next().await.unwrap());
-    ///    assert_eq!(3, rx.next().await.unwrap());
-    ///    assert_eq!(4, rx.next().await.unwrap());
-    ///    assert_eq!(5, rx.next().await.unwrap());
-    ///
-    ///    // The merged stream is consumed
-    ///    assert!(rx.next().await.is_none());
-    /// }
-    /// ```
-    fn merge<U>(self, other: U) -> Merge<Self, U>
-    where
-        U: Stream<Item = Self::Item>,
-        Self: Sized,
-    {
-        Merge::new(self, other)
-    }
-
-    /// Filters the values produced by this stream according to the provided
-    /// predicate.
-    ///
-    /// As values of this stream are made available, the provided predicate `f`
-    /// will be run against them. If the predicate
-    /// resolves to `true`, then the stream will yield the value, but if the
-    /// predicate resolves to `false`, then the value
-    /// will be discarded and the next value will be produced.
-    ///
-    /// Note that this function consumes the stream passed into it and returns a
-    /// wrapped version of it, similar to [`Iterator::filter`] method in the
-    /// standard library.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let stream = stream::iter(1..=8);
-    /// let mut evens = stream.filter(|x| x % 2 == 0);
-    ///
-    /// assert_eq!(Some(2), evens.next().await);
-    /// assert_eq!(Some(4), evens.next().await);
-    /// assert_eq!(Some(6), evens.next().await);
-    /// assert_eq!(Some(8), evens.next().await);
-    /// assert_eq!(None, evens.next().await);
-    /// # }
-    /// ```
-    fn filter<F>(self, f: F) -> Filter<Self, F>
-    where
-        F: FnMut(&Self::Item) -> bool,
-        Self: Sized,
-    {
-        Filter::new(self, f)
-    }
-
-    /// Filters the values produced by this stream while simultaneously mapping
-    /// them to a different type according to the provided closure.
-    ///
-    /// As values of this stream are made available, the provided function will
-    /// be run on them. If the predicate `f` resolves to
-    /// [`Some(item)`](Some) then the stream will yield the value `item`, but if
-    /// it resolves to [`None`], then the value will be skipped.
-    ///
-    /// Note that this function consumes the stream passed into it and returns a
-    /// wrapped version of it, similar to [`Iterator::filter_map`] method in the
-    /// standard library.
-    ///
-    /// # Examples
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let stream = stream::iter(1..=8);
-    /// let mut evens = stream.filter_map(|x| {
-    ///     if x % 2 == 0 { Some(x + 1) } else { None }
-    /// });
-    ///
-    /// assert_eq!(Some(3), evens.next().await);
-    /// assert_eq!(Some(5), evens.next().await);
-    /// assert_eq!(Some(7), evens.next().await);
-    /// assert_eq!(Some(9), evens.next().await);
-    /// assert_eq!(None, evens.next().await);
-    /// # }
-    /// ```
-    fn filter_map<T, F>(self, f: F) -> FilterMap<Self, F>
-    where
-        F: FnMut(Self::Item) -> Option<T>,
-        Self: Sized,
-    {
-        FilterMap::new(self, f)
-    }
-
-    /// Creates a stream which ends after the first `None`.
-    ///
-    /// After a stream returns `None`, behavior is undefined. Future calls to
-    /// `poll_next` may or may not return `Some(T)` again or they may panic.
-    /// `fuse()` adapts a stream, ensuring that after `None` is given, it will
-    /// return `None` forever.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use tokio_stream::{Stream, StreamExt};
-    ///
-    /// use std::pin::Pin;
-    /// use std::task::{Context, Poll};
-    ///
-    /// // a stream which alternates between Some and None
-    /// struct Alternate {
-    ///     state: i32,
-    /// }
-    ///
-    /// impl Stream for Alternate {
-    ///     type Item = i32;
-    ///
-    ///     fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<i32>> {
-    ///         let val = self.state;
-    ///         self.state = self.state + 1;
-    ///
-    ///         // if it's even, Some(i32), else None
-    ///         if val % 2 == 0 {
-    ///             Poll::Ready(Some(val))
-    ///         } else {
-    ///             Poll::Ready(None)
-    ///         }
-    ///     }
-    /// }
-    ///
-    /// #[tokio::main]
-    /// async fn main() {
-    ///     let mut stream = Alternate { state: 0 };
-    ///
-    ///     // the stream goes back and forth
-    ///     assert_eq!(stream.next().await, Some(0));
-    ///     assert_eq!(stream.next().await, None);
-    ///     assert_eq!(stream.next().await, Some(2));
-    ///     assert_eq!(stream.next().await, None);
-    ///
-    ///     // however, once it is fused
-    ///     let mut stream = stream.fuse();
-    ///
-    ///     assert_eq!(stream.next().await, Some(4));
-    ///     assert_eq!(stream.next().await, None);
-    ///
-    ///     // it will always return `None` after the first time.
-    ///     assert_eq!(stream.next().await, None);
-    ///     assert_eq!(stream.next().await, None);
-    ///     assert_eq!(stream.next().await, None);
-    /// }
-    /// ```
-    fn fuse(self) -> Fuse<Self>
-    where
-        Self: Sized,
-    {
-        Fuse::new(self)
-    }
-
-    /// Creates a new stream of at most `n` items of the underlying stream.
-    ///
-    /// Once `n` items have been yielded from this stream then it will always
-    /// return that the stream is done.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let mut stream = stream::iter(1..=10).take(3);
-    ///
-    /// assert_eq!(Some(1), stream.next().await);
-    /// assert_eq!(Some(2), stream.next().await);
-    /// assert_eq!(Some(3), stream.next().await);
-    /// assert_eq!(None, stream.next().await);
-    /// # }
-    /// ```
-    fn take(self, n: usize) -> Take<Self>
-    where
-        Self: Sized,
-    {
-        Take::new(self, n)
-    }
-
-    /// Take elements from this stream while the provided predicate
-    /// resolves to `true`.
-    ///
-    /// This function, like `Iterator::take_while`, will take elements from the
-    /// stream until the predicate `f` resolves to `false`. Once one element
-    /// returns false it will always return that the stream is done.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let mut stream = stream::iter(1..=10).take_while(|x| *x <= 3);
-    ///
-    /// assert_eq!(Some(1), stream.next().await);
-    /// assert_eq!(Some(2), stream.next().await);
-    /// assert_eq!(Some(3), stream.next().await);
-    /// assert_eq!(None, stream.next().await);
-    /// # }
-    /// ```
-    fn take_while<F>(self, f: F) -> TakeWhile<Self, F>
-    where
-        F: FnMut(&Self::Item) -> bool,
-        Self: Sized,
-    {
-        TakeWhile::new(self, f)
-    }
-
-    /// Creates a new stream that will skip the `n` first items of the
-    /// underlying stream.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let mut stream = stream::iter(1..=10).skip(7);
-    ///
-    /// assert_eq!(Some(8), stream.next().await);
-    /// assert_eq!(Some(9), stream.next().await);
-    /// assert_eq!(Some(10), stream.next().await);
-    /// assert_eq!(None, stream.next().await);
-    /// # }
-    /// ```
-    fn skip(self, n: usize) -> Skip<Self>
-    where
-        Self: Sized,
-    {
-        Skip::new(self, n)
-    }
-
-    /// Skip elements from the underlying stream while the provided predicate
-    /// resolves to `true`.
-    ///
-    /// This function, like [`Iterator::skip_while`], will ignore elemets from the
-    /// stream until the predicate `f` resolves to `false`. Once one element
-    /// returns false, the rest of the elements will be yielded.
-    ///
-    /// [`Iterator::skip_while`]: std::iter::Iterator::skip_while()
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    /// let mut stream = stream::iter(vec![1,2,3,4,1]).skip_while(|x| *x < 3);
-    ///
-    /// assert_eq!(Some(3), stream.next().await);
-    /// assert_eq!(Some(4), stream.next().await);
-    /// assert_eq!(Some(1), stream.next().await);
-    /// assert_eq!(None, stream.next().await);
-    /// # }
-    /// ```
-    fn skip_while<F>(self, f: F) -> SkipWhile<Self, F>
-    where
-        F: FnMut(&Self::Item) -> bool,
-        Self: Sized,
-    {
-        SkipWhile::new(self, f)
-    }
-
-    /// Tests if every element of the stream matches a predicate.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn all<F>(&mut self, f: F) -> bool;
-    /// ```
-    ///
-    /// `all()` takes a closure that returns `true` or `false`. It applies
-    /// this closure to each element of the stream, and if they all return
-    /// `true`, then so does `all`. If any of them return `false`, it
-    /// returns `false`. An empty stream returns `true`.
-    ///
-    /// `all()` is short-circuiting; in other words, it will stop processing
-    /// as soon as it finds a `false`, given that no matter what else happens,
-    /// the result will also be `false`.
-    ///
-    /// An empty stream returns `true`.
-    ///
-    /// # Examples
-    ///
-    /// Basic usage:
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let a = [1, 2, 3];
-    ///
-    /// assert!(stream::iter(&a).all(|&x| x > 0).await);
-    ///
-    /// assert!(!stream::iter(&a).all(|&x| x > 2).await);
-    /// # }
-    /// ```
-    ///
-    /// Stopping at the first `false`:
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let a = [1, 2, 3];
-    ///
-    /// let mut iter = stream::iter(&a);
-    ///
-    /// assert!(!iter.all(|&x| x != 2).await);
-    ///
-    /// // we can still use `iter`, as there are more elements.
-    /// assert_eq!(iter.next().await, Some(&3));
-    /// # }
-    /// ```
-    fn all<F>(&mut self, f: F) -> AllFuture<'_, Self, F>
-    where
-        Self: Unpin,
-        F: FnMut(Self::Item) -> bool,
-    {
-        AllFuture::new(self, f)
-    }
-
-    /// Tests if any element of the stream matches a predicate.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn any<F>(&mut self, f: F) -> bool;
-    /// ```
-    ///
-    /// `any()` takes a closure that returns `true` or `false`. It applies
-    /// this closure to each element of the stream, and if any of them return
-    /// `true`, then so does `any()`. If they all return `false`, it
-    /// returns `false`.
-    ///
-    /// `any()` is short-circuiting; in other words, it will stop processing
-    /// as soon as it finds a `true`, given that no matter what else happens,
-    /// the result will also be `true`.
-    ///
-    /// An empty stream returns `false`.
-    ///
-    /// Basic usage:
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let a = [1, 2, 3];
-    ///
-    /// assert!(stream::iter(&a).any(|&x| x > 0).await);
-    ///
-    /// assert!(!stream::iter(&a).any(|&x| x > 5).await);
-    /// # }
-    /// ```
-    ///
-    /// Stopping at the first `true`:
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// let a = [1, 2, 3];
-    ///
-    /// let mut iter = stream::iter(&a);
-    ///
-    /// assert!(iter.any(|&x| x != 2).await);
-    ///
-    /// // we can still use `iter`, as there are more elements.
-    /// assert_eq!(iter.next().await, Some(&2));
-    /// # }
-    /// ```
-    fn any<F>(&mut self, f: F) -> AnyFuture<'_, Self, F>
-    where
-        Self: Unpin,
-        F: FnMut(Self::Item) -> bool,
-    {
-        AnyFuture::new(self, f)
-    }
-
-    /// Combine two streams into one by first returning all values from the
-    /// first stream then all values from the second stream.
-    ///
-    /// As long as `self` still has values to emit, no values from `other` are
-    /// emitted, even if some are ready.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// #[tokio::main]
-    /// async fn main() {
-    ///     let one = stream::iter(vec![1, 2, 3]);
-    ///     let two = stream::iter(vec![4, 5, 6]);
-    ///
-    ///     let mut stream = one.chain(two);
-    ///
-    ///     assert_eq!(stream.next().await, Some(1));
-    ///     assert_eq!(stream.next().await, Some(2));
-    ///     assert_eq!(stream.next().await, Some(3));
-    ///     assert_eq!(stream.next().await, Some(4));
-    ///     assert_eq!(stream.next().await, Some(5));
-    ///     assert_eq!(stream.next().await, Some(6));
-    ///     assert_eq!(stream.next().await, None);
-    /// }
-    /// ```
-    fn chain<U>(self, other: U) -> Chain<Self, U>
-    where
-        U: Stream<Item = Self::Item>,
-        Self: Sized,
-    {
-        Chain::new(self, other)
-    }
-
-    /// A combinator that applies a function to every element in a stream
-    /// producing a single, final value.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn fold<B, F>(self, init: B, f: F) -> B;
-    /// ```
-    ///
-    /// # Examples
-    /// Basic usage:
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, *};
-    ///
-    /// let s = stream::iter(vec![1u8, 2, 3]);
-    /// let sum = s.fold(0, |acc, x| acc + x).await;
-    ///
-    /// assert_eq!(sum, 6);
-    /// # }
-    /// ```
-    fn fold<B, F>(self, init: B, f: F) -> FoldFuture<Self, B, F>
-    where
-        Self: Sized,
-        F: FnMut(B, Self::Item) -> B,
-    {
-        FoldFuture::new(self, init, f)
-    }
-
-    /// Drain stream pushing all emitted values into a collection.
-    ///
-    /// Equivalent to:
-    ///
-    /// ```ignore
-    /// async fn collect<T>(self) -> T;
-    /// ```
-    ///
-    /// `collect` streams all values, awaiting as needed. Values are pushed into
-    /// a collection. A number of different target collection types are
-    /// supported, including [`Vec`](std::vec::Vec),
-    /// [`String`](std::string::String), and [`Bytes`].
-    ///
-    /// [`Bytes`]: https://docs.rs/bytes/0.6.0/bytes/struct.Bytes.html
-    ///
-    /// # `Result`
-    ///
-    /// `collect()` can also be used with streams of type `Result<T, E>` where
-    /// `T: FromStream<_>`. In this case, `collect()` will stream as long as
-    /// values yielded from the stream are `Ok(_)`. If `Err(_)` is encountered,
-    /// streaming is terminated and `collect()` returns the `Err`.
-    ///
-    /// # Notes
-    ///
-    /// `FromStream` is currently a sealed trait. Stabilization is pending
-    /// enhancements to the Rust language.
-    ///
-    /// # Examples
-    ///
-    /// Basic usage:
-    ///
-    /// ```
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// #[tokio::main]
-    /// async fn main() {
-    ///     let doubled: Vec<i32> =
-    ///         stream::iter(vec![1, 2, 3])
-    ///             .map(|x| x * 2)
-    ///             .collect()
-    ///             .await;
-    ///
-    ///     assert_eq!(vec![2, 4, 6], doubled);
-    /// }
-    /// ```
-    ///
-    /// Collecting a stream of `Result` values
-    ///
-    /// ```
-    /// use tokio_stream::{self as stream, StreamExt};
-    ///
-    /// #[tokio::main]
-    /// async fn main() {
-    ///     // A stream containing only `Ok` values will be collected
-    ///     let values: Result<Vec<i32>, &str> =
-    ///         stream::iter(vec![Ok(1), Ok(2), Ok(3)])
-    ///             .collect()
-    ///             .await;
-    ///
-    ///     assert_eq!(Ok(vec![1, 2, 3]), values);
-    ///
-    ///     // A stream containing `Err` values will return the first error.
-    ///     let results = vec![Ok(1), Err("no"), Ok(2), Ok(3), Err("nein")];
-    ///
-    ///     let values: Result<Vec<i32>, &str> =
-    ///         stream::iter(results)
-    ///             .collect()
-    ///             .await;
-    ///
-    ///     assert_eq!(Err("no"), values);
-    /// }
-    /// ```
-    fn collect<T>(self) -> Collect<Self, T>
-    where
-        T: FromStream<Self::Item>,
-        Self: Sized,
-    {
-        Collect::new(self)
-    }
-
-    /// Applies a per-item timeout to the passed stream.
-    ///
-    /// `timeout()` takes a `Duration` that represents the maximum amount of
-    /// time each element of the stream has to complete before timing out.
-    ///
-    /// If the wrapped stream yields a value before the deadline is reached, the
-    /// value is returned. Otherwise, an error is returned. The caller may decide
-    /// to continue consuming the stream and will eventually get the next source
-    /// stream value once it becomes available.
-    ///
-    /// # Notes
-    ///
-    /// This function consumes the stream passed into it and returns a
-    /// wrapped version of it.
-    ///
-    /// Polling the returned stream will continue to poll the inner stream even
-    /// if one or more items time out.
-    ///
-    /// # Examples
-    ///
-    /// Suppose we have a stream `int_stream` that yields 3 numbers (1, 2, 3):
-    ///
-    /// ```
-    /// # #[tokio::main]
-    /// # async fn main() {
-    /// use tokio_stream::{self as stream, StreamExt};
-    /// use std::time::Duration;
-    /// # let int_stream = stream::iter(1..=3);
-    ///
-    /// let int_stream = int_stream.timeout(Duration::from_secs(1));
-    /// tokio::pin!(int_stream);
-    ///
-    /// // When no items time out, we get the 3 elements in succession:
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(2)));
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(3)));
-    /// assert_eq!(int_stream.try_next().await, Ok(None));
-    ///
-    /// // If the second item times out, we get an error and continue polling the stream:
-    /// # let mut int_stream = stream::iter(vec![Ok(1), Err(()), Ok(2), Ok(3)]);
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
-    /// assert!(int_stream.try_next().await.is_err());
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(2)));
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(3)));
-    /// assert_eq!(int_stream.try_next().await, Ok(None));
-    ///
-    /// // If we want to stop consuming the source stream the first time an
-    /// // element times out, we can use the `take_while` operator:
-    /// # let int_stream = stream::iter(vec![Ok(1), Err(()), Ok(2), Ok(3)]);
-    /// let mut int_stream = int_stream.take_while(Result::is_ok);
-    ///
-    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
-    /// assert_eq!(int_stream.try_next().await, Ok(None));
-    /// # }
-    /// ```
-    #[cfg(all(feature = "time"))]
-    #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
-    fn timeout(self, duration: Duration) -> Timeout<Self>
-    where
-        Self: Sized,
-    {
-        Timeout::new(self, duration)
-    }
-
-    /// Slows down a stream by enforcing a delay between items.
-    ///
-    /// # Example
-    ///
-    /// Create a throttled stream.
-    /// ```rust,no_run
-    /// use std::time::Duration;
-    /// use tokio_stream::StreamExt;
-    ///
-    /// # async fn dox() {
-    /// let item_stream = futures::stream::repeat("one").throttle(Duration::from_secs(2));
-    /// tokio::pin!(item_stream);
-    ///
-    /// loop {
-    ///     // The string will be produced at most every 2 seconds
-    ///     println!("{:?}", item_stream.next().await);
-    /// }
-    /// # }
-    /// ```
-    #[cfg(all(feature = "time"))]
-    #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
-    fn throttle(self, duration: Duration) -> Throttle<Self>
-    where
-        Self: Sized,
-    {
-        throttle(duration, self)
-    }
-}
-
-impl<St: ?Sized> StreamExt for St where St: Stream {}
-
-/// Merge the size hints from two streams.
-fn merge_size_hints(
-    (left_low, left_high): (usize, Option<usize>),
-    (right_low, right_hign): (usize, Option<usize>),
-) -> (usize, Option<usize>) {
-    let low = left_low.saturating_add(right_low);
-    let high = match (left_high, right_hign) {
-        (Some(h1), Some(h2)) => h1.checked_add(h2),
-        _ => None,
-    };
-    (low, high)
-}

tokio-stream/src/macros.rs

@@ -1,3 +1,33 @@
+macro_rules! cfg_fs {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "fs")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
+            $item
+        )*
+    }
+}
+
+macro_rules! cfg_io_util {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "io-util")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "io-util")))]
+            $item
+        )*
+    }
+}
+
+macro_rules! cfg_net {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "net")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "net")))]
+            $item
+        )*
+    }
+}
+
 macro_rules! cfg_time {
     ($($item:item)*) => {
         $(
@@ -8,6 +38,26 @@ macro_rules! cfg_time {
     }
 }
 
+macro_rules! cfg_sync {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "sync")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
+            $item
+        )*
+    }
+}
+
+macro_rules! cfg_signal {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "signal")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "signal")))]
+            $item
+        )*
+    }
+}
+
 macro_rules! ready {
     ($e:expr $(,)?) => {
         match $e {

tokio-stream/src/stream_ext.rs

@@ -0,0 +1,917 @@
+use futures_core::Stream;
+
+mod all;
+use all::AllFuture;
+
+mod any;
+use any::AnyFuture;
+
+mod chain;
+use chain::Chain;
+
+pub(crate) mod collect;
+use collect::{Collect, FromStream};
+
+mod filter;
+use filter::Filter;
+
+mod filter_map;
+use filter_map::FilterMap;
+
+mod fold;
+use fold::FoldFuture;
+
+mod fuse;
+use fuse::Fuse;
+
+mod map;
+use map::Map;
+
+mod merge;
+use merge::Merge;
+
+mod next;
+use next::Next;
+
+mod skip;
+use skip::Skip;
+
+mod skip_while;
+use skip_while::SkipWhile;
+
+mod try_next;
+use try_next::TryNext;
+
+mod take;
+use take::Take;
+
+mod take_while;
+use take_while::TakeWhile;
+
+cfg_time! {
+    mod timeout;
+    use timeout::Timeout;
+    use tokio::time::Duration;
+    mod throttle;
+    use throttle::{throttle, Throttle};
+}
+
+/// An extension trait for the [`Stream`] trait that provides a variety of
+/// convenient combinator functions.
+///
+/// Be aware that the `Stream` trait in Tokio is a re-export of the trait found
+/// in the [futures] crate, however both Tokio and futures provide separate
+/// `StreamExt` utility traits, and some utilities are only available on one of
+/// these traits. Click [here][futures-StreamExt] to see the other `StreamExt`
+/// trait in the futures crate.
+///
+/// If you need utilities from both `StreamExt` traits, you should prefer to
+/// import one of them, and use the other through the fully qualified call
+/// syntax. For example:
+/// ```
+/// // import one of the traits:
+/// use futures::stream::StreamExt;
+/// # #[tokio::main(flavor = "current_thread")]
+/// # async fn main() {
+///
+/// let a = tokio_stream::iter(vec![1, 3, 5]);
+/// let b = tokio_stream::iter(vec![2, 4, 6]);
+///
+/// // use the fully qualified call syntax for the other trait:
+/// let merged = tokio_stream::StreamExt::merge(a, b);
+///
+/// // use normal call notation for futures::stream::StreamExt::collect
+/// let output: Vec<_> = merged.collect().await;
+/// assert_eq!(output, vec![1, 2, 3, 4, 5, 6]);
+/// # }
+/// ```
+///
+/// [`Stream`]: crate::Stream
+/// [futures]: https://docs.rs/futures
+/// [futures-StreamExt]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html
+pub trait StreamExt: Stream {
+    /// Consumes and returns the next value in the stream or `None` if the
+    /// stream is finished.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn next(&mut self) -> Option<Self::Item>;
+    /// ```
+    ///
+    /// Note that because `next` doesn't take ownership over the stream,
+    /// the [`Stream`] type must be [`Unpin`]. If you want to use `next` with a
+    /// [`!Unpin`](Unpin) stream, you'll first have to pin the stream. This can
+    /// be done by boxing the stream using [`Box::pin`] or
+    /// pinning it to the stack using the `pin_mut!` macro from the `pin_utils`
+    /// crate.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let mut stream = stream::iter(1..=3);
+    ///
+    /// assert_eq!(stream.next().await, Some(1));
+    /// assert_eq!(stream.next().await, Some(2));
+    /// assert_eq!(stream.next().await, Some(3));
+    /// assert_eq!(stream.next().await, None);
+    /// # }
+    /// ```
+    fn next(&mut self) -> Next<'_, Self>
+    where
+        Self: Unpin,
+    {
+        Next::new(self)
+    }
+
+    /// Consumes and returns the next item in the stream. If an error is
+    /// encountered before the next item, the error is returned instead.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn try_next(&mut self) -> Result<Option<T>, E>;
+    /// ```
+    ///
+    /// This is similar to the [`next`](StreamExt::next) combinator,
+    /// but returns a [`Result<Option<T>, E>`](Result) rather than
+    /// an [`Option<Result<T, E>>`](Option), making for easy use
+    /// with the [`?`](std::ops::Try) operator.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let mut stream = stream::iter(vec![Ok(1), Ok(2), Err("nope")]);
+    ///
+    /// assert_eq!(stream.try_next().await, Ok(Some(1)));
+    /// assert_eq!(stream.try_next().await, Ok(Some(2)));
+    /// assert_eq!(stream.try_next().await, Err("nope"));
+    /// # }
+    /// ```
+    fn try_next<T, E>(&mut self) -> TryNext<'_, Self>
+    where
+        Self: Stream<Item = Result<T, E>> + Unpin,
+    {
+        TryNext::new(self)
+    }
+
+    /// Maps this stream's items to a different type, returning a new stream of
+    /// the resulting type.
+    ///
+    /// The provided closure is executed over all elements of this stream as
+    /// they are made available. It is executed inline with calls to
+    /// [`poll_next`](Stream::poll_next).
+    ///
+    /// Note that this function consumes the stream passed into it and returns a
+    /// wrapped version of it, similar to the existing `map` methods in the
+    /// standard library.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let stream = stream::iter(1..=3);
+    /// let mut stream = stream.map(|x| x + 3);
+    ///
+    /// assert_eq!(stream.next().await, Some(4));
+    /// assert_eq!(stream.next().await, Some(5));
+    /// assert_eq!(stream.next().await, Some(6));
+    /// # }
+    /// ```
+    fn map<T, F>(self, f: F) -> Map<Self, F>
+    where
+        F: FnMut(Self::Item) -> T,
+        Self: Sized,
+    {
+        Map::new(self, f)
+    }
+
+    /// Combine two streams into one by interleaving the output of both as it
+    /// is produced.
+    ///
+    /// Values are produced from the merged stream in the order they arrive from
+    /// the two source streams. If both source streams provide values
+    /// simultaneously, the merge stream alternates between them. This provides
+    /// some level of fairness. You should not chain calls to `merge`, as this
+    /// will break the fairness of the merging.
+    ///
+    /// The merged stream completes once **both** source streams complete. When
+    /// one source stream completes before the other, the merge stream
+    /// exclusively polls the remaining stream.
+    ///
+    /// For merging multiple streams, consider using [`StreamMap`] instead.
+    ///
+    /// [`StreamMap`]: crate::StreamMap
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_stream::{StreamExt, Stream};
+    /// use tokio::sync::mpsc;
+    /// use tokio::time;
+    ///
+    /// use std::time::Duration;
+    /// use std::pin::Pin;
+    ///
+    /// # /*
+    /// #[tokio::main]
+    /// # */
+    /// # #[tokio::main(flavor = "current_thread")]
+    /// async fn main() {
+    /// # time::pause();
+    ///     let (tx1, mut rx1) = mpsc::channel::<usize>(10);
+    ///     let (tx2, mut rx2) = mpsc::channel::<usize>(10);
+    ///
+    ///     // Convert the channels to a `Stream`.
+    ///     let rx1 = Box::pin(async_stream::stream! {
+    ///           while let Some(item) = rx1.recv().await {
+    ///               yield item;
+    ///           }
+    ///     }) as Pin<Box<dyn Stream<Item = usize> + Send>>;
+    ///
+    ///     let rx2 = Box::pin(async_stream::stream! {
+    ///           while let Some(item) = rx2.recv().await {
+    ///               yield item;
+    ///           }
+    ///     }) as Pin<Box<dyn Stream<Item = usize> + Send>>;
+    ///
+    ///     let mut rx = rx1.merge(rx2);
+    ///
+    ///     tokio::spawn(async move {
+    ///         // Send some values immediately
+    ///         tx1.send(1).await.unwrap();
+    ///         tx1.send(2).await.unwrap();
+    ///
+    ///         // Let the other task send values
+    ///         time::sleep(Duration::from_millis(20)).await;
+    ///
+    ///         tx1.send(4).await.unwrap();
+    ///     });
+    ///
+    ///     tokio::spawn(async move {
+    ///         // Wait for the first task to send values
+    ///         time::sleep(Duration::from_millis(5)).await;
+    ///
+    ///         tx2.send(3).await.unwrap();
+    ///
+    ///         time::sleep(Duration::from_millis(25)).await;
+    ///
+    ///         // Send the final value
+    ///         tx2.send(5).await.unwrap();
+    ///     });
+    ///
+    ///    assert_eq!(1, rx.next().await.unwrap());
+    ///    assert_eq!(2, rx.next().await.unwrap());
+    ///    assert_eq!(3, rx.next().await.unwrap());
+    ///    assert_eq!(4, rx.next().await.unwrap());
+    ///    assert_eq!(5, rx.next().await.unwrap());
+    ///
+    ///    // The merged stream is consumed
+    ///    assert!(rx.next().await.is_none());
+    /// }
+    /// ```
+    fn merge<U>(self, other: U) -> Merge<Self, U>
+    where
+        U: Stream<Item = Self::Item>,
+        Self: Sized,
+    {
+        Merge::new(self, other)
+    }
+
+    /// Filters the values produced by this stream according to the provided
+    /// predicate.
+    ///
+    /// As values of this stream are made available, the provided predicate `f`
+    /// will be run against them. If the predicate
+    /// resolves to `true`, then the stream will yield the value, but if the
+    /// predicate resolves to `false`, then the value
+    /// will be discarded and the next value will be produced.
+    ///
+    /// Note that this function consumes the stream passed into it and returns a
+    /// wrapped version of it, similar to [`Iterator::filter`] method in the
+    /// standard library.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let stream = stream::iter(1..=8);
+    /// let mut evens = stream.filter(|x| x % 2 == 0);
+    ///
+    /// assert_eq!(Some(2), evens.next().await);
+    /// assert_eq!(Some(4), evens.next().await);
+    /// assert_eq!(Some(6), evens.next().await);
+    /// assert_eq!(Some(8), evens.next().await);
+    /// assert_eq!(None, evens.next().await);
+    /// # }
+    /// ```
+    fn filter<F>(self, f: F) -> Filter<Self, F>
+    where
+        F: FnMut(&Self::Item) -> bool,
+        Self: Sized,
+    {
+        Filter::new(self, f)
+    }
+
+    /// Filters the values produced by this stream while simultaneously mapping
+    /// them to a different type according to the provided closure.
+    ///
+    /// As values of this stream are made available, the provided function will
+    /// be run on them. If the predicate `f` resolves to
+    /// [`Some(item)`](Some) then the stream will yield the value `item`, but if
+    /// it resolves to [`None`], then the value will be skipped.
+    ///
+    /// Note that this function consumes the stream passed into it and returns a
+    /// wrapped version of it, similar to [`Iterator::filter_map`] method in the
+    /// standard library.
+    ///
+    /// # Examples
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let stream = stream::iter(1..=8);
+    /// let mut evens = stream.filter_map(|x| {
+    ///     if x % 2 == 0 { Some(x + 1) } else { None }
+    /// });
+    ///
+    /// assert_eq!(Some(3), evens.next().await);
+    /// assert_eq!(Some(5), evens.next().await);
+    /// assert_eq!(Some(7), evens.next().await);
+    /// assert_eq!(Some(9), evens.next().await);
+    /// assert_eq!(None, evens.next().await);
+    /// # }
+    /// ```
+    fn filter_map<T, F>(self, f: F) -> FilterMap<Self, F>
+    where
+        F: FnMut(Self::Item) -> Option<T>,
+        Self: Sized,
+    {
+        FilterMap::new(self, f)
+    }
+
+    /// Creates a stream which ends after the first `None`.
+    ///
+    /// After a stream returns `None`, behavior is undefined. Future calls to
+    /// `poll_next` may or may not return `Some(T)` again or they may panic.
+    /// `fuse()` adapts a stream, ensuring that after `None` is given, it will
+    /// return `None` forever.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_stream::{Stream, StreamExt};
+    ///
+    /// use std::pin::Pin;
+    /// use std::task::{Context, Poll};
+    ///
+    /// // a stream which alternates between Some and None
+    /// struct Alternate {
+    ///     state: i32,
+    /// }
+    ///
+    /// impl Stream for Alternate {
+    ///     type Item = i32;
+    ///
+    ///     fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<i32>> {
+    ///         let val = self.state;
+    ///         self.state = self.state + 1;
+    ///
+    ///         // if it's even, Some(i32), else None
+    ///         if val % 2 == 0 {
+    ///             Poll::Ready(Some(val))
+    ///         } else {
+    ///             Poll::Ready(None)
+    ///         }
+    ///     }
+    /// }
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     let mut stream = Alternate { state: 0 };
+    ///
+    ///     // the stream goes back and forth
+    ///     assert_eq!(stream.next().await, Some(0));
+    ///     assert_eq!(stream.next().await, None);
+    ///     assert_eq!(stream.next().await, Some(2));
+    ///     assert_eq!(stream.next().await, None);
+    ///
+    ///     // however, once it is fused
+    ///     let mut stream = stream.fuse();
+    ///
+    ///     assert_eq!(stream.next().await, Some(4));
+    ///     assert_eq!(stream.next().await, None);
+    ///
+    ///     // it will always return `None` after the first time.
+    ///     assert_eq!(stream.next().await, None);
+    ///     assert_eq!(stream.next().await, None);
+    ///     assert_eq!(stream.next().await, None);
+    /// }
+    /// ```
+    fn fuse(self) -> Fuse<Self>
+    where
+        Self: Sized,
+    {
+        Fuse::new(self)
+    }
+
+    /// Creates a new stream of at most `n` items of the underlying stream.
+    ///
+    /// Once `n` items have been yielded from this stream then it will always
+    /// return that the stream is done.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let mut stream = stream::iter(1..=10).take(3);
+    ///
+    /// assert_eq!(Some(1), stream.next().await);
+    /// assert_eq!(Some(2), stream.next().await);
+    /// assert_eq!(Some(3), stream.next().await);
+    /// assert_eq!(None, stream.next().await);
+    /// # }
+    /// ```
+    fn take(self, n: usize) -> Take<Self>
+    where
+        Self: Sized,
+    {
+        Take::new(self, n)
+    }
+
+    /// Take elements from this stream while the provided predicate
+    /// resolves to `true`.
+    ///
+    /// This function, like `Iterator::take_while`, will take elements from the
+    /// stream until the predicate `f` resolves to `false`. Once one element
+    /// returns false it will always return that the stream is done.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let mut stream = stream::iter(1..=10).take_while(|x| *x <= 3);
+    ///
+    /// assert_eq!(Some(1), stream.next().await);
+    /// assert_eq!(Some(2), stream.next().await);
+    /// assert_eq!(Some(3), stream.next().await);
+    /// assert_eq!(None, stream.next().await);
+    /// # }
+    /// ```
+    fn take_while<F>(self, f: F) -> TakeWhile<Self, F>
+    where
+        F: FnMut(&Self::Item) -> bool,
+        Self: Sized,
+    {
+        TakeWhile::new(self, f)
+    }
+
+    /// Creates a new stream that will skip the `n` first items of the
+    /// underlying stream.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let mut stream = stream::iter(1..=10).skip(7);
+    ///
+    /// assert_eq!(Some(8), stream.next().await);
+    /// assert_eq!(Some(9), stream.next().await);
+    /// assert_eq!(Some(10), stream.next().await);
+    /// assert_eq!(None, stream.next().await);
+    /// # }
+    /// ```
+    fn skip(self, n: usize) -> Skip<Self>
+    where
+        Self: Sized,
+    {
+        Skip::new(self, n)
+    }
+
+    /// Skip elements from the underlying stream while the provided predicate
+    /// resolves to `true`.
+    ///
+    /// This function, like [`Iterator::skip_while`], will ignore elemets from the
+    /// stream until the predicate `f` resolves to `false`. Once one element
+    /// returns false, the rest of the elements will be yielded.
+    ///
+    /// [`Iterator::skip_while`]: std::iter::Iterator::skip_while()
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    /// let mut stream = stream::iter(vec![1,2,3,4,1]).skip_while(|x| *x < 3);
+    ///
+    /// assert_eq!(Some(3), stream.next().await);
+    /// assert_eq!(Some(4), stream.next().await);
+    /// assert_eq!(Some(1), stream.next().await);
+    /// assert_eq!(None, stream.next().await);
+    /// # }
+    /// ```
+    fn skip_while<F>(self, f: F) -> SkipWhile<Self, F>
+    where
+        F: FnMut(&Self::Item) -> bool,
+        Self: Sized,
+    {
+        SkipWhile::new(self, f)
+    }
+
+    /// Tests if every element of the stream matches a predicate.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn all<F>(&mut self, f: F) -> bool;
+    /// ```
+    ///
+    /// `all()` takes a closure that returns `true` or `false`. It applies
+    /// this closure to each element of the stream, and if they all return
+    /// `true`, then so does `all`. If any of them return `false`, it
+    /// returns `false`. An empty stream returns `true`.
+    ///
+    /// `all()` is short-circuiting; in other words, it will stop processing
+    /// as soon as it finds a `false`, given that no matter what else happens,
+    /// the result will also be `false`.
+    ///
+    /// An empty stream returns `true`.
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let a = [1, 2, 3];
+    ///
+    /// assert!(stream::iter(&a).all(|&x| x > 0).await);
+    ///
+    /// assert!(!stream::iter(&a).all(|&x| x > 2).await);
+    /// # }
+    /// ```
+    ///
+    /// Stopping at the first `false`:
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let a = [1, 2, 3];
+    ///
+    /// let mut iter = stream::iter(&a);
+    ///
+    /// assert!(!iter.all(|&x| x != 2).await);
+    ///
+    /// // we can still use `iter`, as there are more elements.
+    /// assert_eq!(iter.next().await, Some(&3));
+    /// # }
+    /// ```
+    fn all<F>(&mut self, f: F) -> AllFuture<'_, Self, F>
+    where
+        Self: Unpin,
+        F: FnMut(Self::Item) -> bool,
+    {
+        AllFuture::new(self, f)
+    }
+
+    /// Tests if any element of the stream matches a predicate.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn any<F>(&mut self, f: F) -> bool;
+    /// ```
+    ///
+    /// `any()` takes a closure that returns `true` or `false`. It applies
+    /// this closure to each element of the stream, and if any of them return
+    /// `true`, then so does `any()`. If they all return `false`, it
+    /// returns `false`.
+    ///
+    /// `any()` is short-circuiting; in other words, it will stop processing
+    /// as soon as it finds a `true`, given that no matter what else happens,
+    /// the result will also be `true`.
+    ///
+    /// An empty stream returns `false`.
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let a = [1, 2, 3];
+    ///
+    /// assert!(stream::iter(&a).any(|&x| x > 0).await);
+    ///
+    /// assert!(!stream::iter(&a).any(|&x| x > 5).await);
+    /// # }
+    /// ```
+    ///
+    /// Stopping at the first `true`:
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// let a = [1, 2, 3];
+    ///
+    /// let mut iter = stream::iter(&a);
+    ///
+    /// assert!(iter.any(|&x| x != 2).await);
+    ///
+    /// // we can still use `iter`, as there are more elements.
+    /// assert_eq!(iter.next().await, Some(&2));
+    /// # }
+    /// ```
+    fn any<F>(&mut self, f: F) -> AnyFuture<'_, Self, F>
+    where
+        Self: Unpin,
+        F: FnMut(Self::Item) -> bool,
+    {
+        AnyFuture::new(self, f)
+    }
+
+    /// Combine two streams into one by first returning all values from the
+    /// first stream then all values from the second stream.
+    ///
+    /// As long as `self` still has values to emit, no values from `other` are
+    /// emitted, even if some are ready.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     let one = stream::iter(vec![1, 2, 3]);
+    ///     let two = stream::iter(vec![4, 5, 6]);
+    ///
+    ///     let mut stream = one.chain(two);
+    ///
+    ///     assert_eq!(stream.next().await, Some(1));
+    ///     assert_eq!(stream.next().await, Some(2));
+    ///     assert_eq!(stream.next().await, Some(3));
+    ///     assert_eq!(stream.next().await, Some(4));
+    ///     assert_eq!(stream.next().await, Some(5));
+    ///     assert_eq!(stream.next().await, Some(6));
+    ///     assert_eq!(stream.next().await, None);
+    /// }
+    /// ```
+    fn chain<U>(self, other: U) -> Chain<Self, U>
+    where
+        U: Stream<Item = Self::Item>,
+        Self: Sized,
+    {
+        Chain::new(self, other)
+    }
+
+    /// A combinator that applies a function to every element in a stream
+    /// producing a single, final value.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn fold<B, F>(self, init: B, f: F) -> B;
+    /// ```
+    ///
+    /// # Examples
+    /// Basic usage:
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, *};
+    ///
+    /// let s = stream::iter(vec![1u8, 2, 3]);
+    /// let sum = s.fold(0, |acc, x| acc + x).await;
+    ///
+    /// assert_eq!(sum, 6);
+    /// # }
+    /// ```
+    fn fold<B, F>(self, init: B, f: F) -> FoldFuture<Self, B, F>
+    where
+        Self: Sized,
+        F: FnMut(B, Self::Item) -> B,
+    {
+        FoldFuture::new(self, init, f)
+    }
+
+    /// Drain stream pushing all emitted values into a collection.
+    ///
+    /// Equivalent to:
+    ///
+    /// ```ignore
+    /// async fn collect<T>(self) -> T;
+    /// ```
+    ///
+    /// `collect` streams all values, awaiting as needed. Values are pushed into
+    /// a collection. A number of different target collection types are
+    /// supported, including [`Vec`](std::vec::Vec),
+    /// [`String`](std::string::String), and [`Bytes`].
+    ///
+    /// [`Bytes`]: https://docs.rs/bytes/0.6.0/bytes/struct.Bytes.html
+    ///
+    /// # `Result`
+    ///
+    /// `collect()` can also be used with streams of type `Result<T, E>` where
+    /// `T: FromStream<_>`. In this case, `collect()` will stream as long as
+    /// values yielded from the stream are `Ok(_)`. If `Err(_)` is encountered,
+    /// streaming is terminated and `collect()` returns the `Err`.
+    ///
+    /// # Notes
+    ///
+    /// `FromStream` is currently a sealed trait. Stabilization is pending
+    /// enhancements to the Rust language.
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     let doubled: Vec<i32> =
+    ///         stream::iter(vec![1, 2, 3])
+    ///             .map(|x| x * 2)
+    ///             .collect()
+    ///             .await;
+    ///
+    ///     assert_eq!(vec![2, 4, 6], doubled);
+    /// }
+    /// ```
+    ///
+    /// Collecting a stream of `Result` values
+    ///
+    /// ```
+    /// use tokio_stream::{self as stream, StreamExt};
+    ///
+    /// #[tokio::main]
+    /// async fn main() {
+    ///     // A stream containing only `Ok` values will be collected
+    ///     let values: Result<Vec<i32>, &str> =
+    ///         stream::iter(vec![Ok(1), Ok(2), Ok(3)])
+    ///             .collect()
+    ///             .await;
+    ///
+    ///     assert_eq!(Ok(vec![1, 2, 3]), values);
+    ///
+    ///     // A stream containing `Err` values will return the first error.
+    ///     let results = vec![Ok(1), Err("no"), Ok(2), Ok(3), Err("nein")];
+    ///
+    ///     let values: Result<Vec<i32>, &str> =
+    ///         stream::iter(results)
+    ///             .collect()
+    ///             .await;
+    ///
+    ///     assert_eq!(Err("no"), values);
+    /// }
+    /// ```
+    fn collect<T>(self) -> Collect<Self, T>
+    where
+        T: FromStream<Self::Item>,
+        Self: Sized,
+    {
+        Collect::new(self)
+    }
+
+    /// Applies a per-item timeout to the passed stream.
+    ///
+    /// `timeout()` takes a `Duration` that represents the maximum amount of
+    /// time each element of the stream has to complete before timing out.
+    ///
+    /// If the wrapped stream yields a value before the deadline is reached, the
+    /// value is returned. Otherwise, an error is returned. The caller may decide
+    /// to continue consuming the stream and will eventually get the next source
+    /// stream value once it becomes available.
+    ///
+    /// # Notes
+    ///
+    /// This function consumes the stream passed into it and returns a
+    /// wrapped version of it.
+    ///
+    /// Polling the returned stream will continue to poll the inner stream even
+    /// if one or more items time out.
+    ///
+    /// # Examples
+    ///
+    /// Suppose we have a stream `int_stream` that yields 3 numbers (1, 2, 3):
+    ///
+    /// ```
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// use tokio_stream::{self as stream, StreamExt};
+    /// use std::time::Duration;
+    /// # let int_stream = stream::iter(1..=3);
+    ///
+    /// let int_stream = int_stream.timeout(Duration::from_secs(1));
+    /// tokio::pin!(int_stream);
+    ///
+    /// // When no items time out, we get the 3 elements in succession:
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(2)));
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(3)));
+    /// assert_eq!(int_stream.try_next().await, Ok(None));
+    ///
+    /// // If the second item times out, we get an error and continue polling the stream:
+    /// # let mut int_stream = stream::iter(vec![Ok(1), Err(()), Ok(2), Ok(3)]);
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
+    /// assert!(int_stream.try_next().await.is_err());
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(2)));
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(3)));
+    /// assert_eq!(int_stream.try_next().await, Ok(None));
+    ///
+    /// // If we want to stop consuming the source stream the first time an
+    /// // element times out, we can use the `take_while` operator:
+    /// # let int_stream = stream::iter(vec![Ok(1), Err(()), Ok(2), Ok(3)]);
+    /// let mut int_stream = int_stream.take_while(Result::is_ok);
+    ///
+    /// assert_eq!(int_stream.try_next().await, Ok(Some(1)));
+    /// assert_eq!(int_stream.try_next().await, Ok(None));
+    /// # }
+    /// ```
+    #[cfg(all(feature = "time"))]
+    #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
+    fn timeout(self, duration: Duration) -> Timeout<Self>
+    where
+        Self: Sized,
+    {
+        Timeout::new(self, duration)
+    }
+
+    /// Slows down a stream by enforcing a delay between items.
+    ///
+    /// # Example
+    ///
+    /// Create a throttled stream.
+    /// ```rust,no_run
+    /// use std::time::Duration;
+    /// use tokio_stream::StreamExt;
+    ///
+    /// # async fn dox() {
+    /// let item_stream = futures::stream::repeat("one").throttle(Duration::from_secs(2));
+    /// tokio::pin!(item_stream);
+    ///
+    /// loop {
+    ///     // The string will be produced at most every 2 seconds
+    ///     println!("{:?}", item_stream.next().await);
+    /// }
+    /// # }
+    /// ```
+    #[cfg(all(feature = "time"))]
+    #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
+    fn throttle(self, duration: Duration) -> Throttle<Self>
+    where
+        Self: Sized,
+    {
+        throttle(duration, self)
+    }
+}
+
+impl<St: ?Sized> StreamExt for St where St: Stream {}
+
+/// Merge the size hints from two streams.
+fn merge_size_hints(
+    (left_low, left_high): (usize, Option<usize>),
+    (right_low, right_hign): (usize, Option<usize>),
+) -> (usize, Option<usize>) {
+    let low = left_low.saturating_add(right_low);
+    let high = match (left_high, right_hign) {
+        (Some(h1), Some(h2)) => h1.checked_add(h2),
+        _ => None,
+    };
+    (low, high)
+}

tokio-stream/src/stream_ext/all.rs

@@ -38,18 +38,21 @@ where
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         let me = self.project();
-        let next = futures_core::ready!(Pin::new(me.stream).poll_next(cx));
+        let mut stream = Pin::new(me.stream);
 
-        match next {
-            Some(v) => {
-                if !(me.f)(v) {
-                    Poll::Ready(false)
-                } else {
-                    cx.waker().wake_by_ref();
-                    Poll::Pending
+        // Take a maximum of 32 items from the stream before yielding.
+        for _ in 0..32 {
+            match futures_core::ready!(stream.as_mut().poll_next(cx)) {
+                Some(v) => {
+                    if !(me.f)(v) {
+                        return Poll::Ready(false);
+                    }
                 }
+                None => return Poll::Ready(true),
             }
-            None => Poll::Ready(true),
         }
+
+        cx.waker().wake_by_ref();
+        Poll::Pending
     }
 }

tokio-stream/src/stream_ext/any.rs

@@ -38,18 +38,21 @@ where
 
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         let me = self.project();
-        let next = futures_core::ready!(Pin::new(me.stream).poll_next(cx));
+        let mut stream = Pin::new(me.stream);
 
-        match next {
-            Some(v) => {
-                if (me.f)(v) {
-                    Poll::Ready(true)
-                } else {
-                    cx.waker().wake_by_ref();
-                    Poll::Pending
+        // Take a maximum of 32 items from the stream before yielding.
+        for _ in 0..32 {
+            match futures_core::ready!(stream.as_mut().poll_next(cx)) {
+                Some(v) => {
+                    if (me.f)(v) {
+                        return Poll::Ready(true);
+                    }
                 }
+                None => return Poll::Ready(false),
             }
-            None => Poll::Ready(false),
         }
+
+        cx.waker().wake_by_ref();
+        Poll::Pending
     }
 }

tokio-stream/src/stream_ext/chain.rs

@@ -1,4 +1,5 @@
-use crate::{Fuse, Stream};
+use crate::stream_ext::Fuse;
+use crate::Stream;
 
 use core::pin::Pin;
 use core::task::{Context, Poll};

tokio-stream/src/stream_ext/merge.rs

@@ -1,4 +1,5 @@
-use crate::{Fuse, Stream};
+use crate::stream_ext::Fuse;
+use crate::Stream;
 
 use core::pin::Pin;
 use core::task::{Context, Poll};

tokio-stream/src/stream_ext/throttle.rs

@@ -23,7 +23,8 @@ where
 }
 
 pin_project! {
-    /// Stream for the [`throttle`](throttle) function.
+    /// Stream for the [`throttle`](throttle) function. This object is `!Unpin`. If you need it to
+    /// implement `Unpin` you can pin your throttle like this: `Box::pin(your_throttle)`.
     #[derive(Debug)]
     #[must_use = "streams do nothing unless polled"]
     pub struct Throttle<T> {

tokio-stream/src/stream_ext/timeout.rs

@@ -1,4 +1,5 @@
-use crate::{Fuse, Stream};
+use crate::stream_ext::Fuse;
+use crate::Stream;
 use tokio::time::{Instant, Sleep};
 
 use core::future::Future;

tokio-stream/src/stream_ext/try_next.rs

@@ -1,4 +1,5 @@
-use crate::{Next, Stream};
+use crate::stream_ext::Next;
+use crate::Stream;
 
 use core::future::Future;
 use core::marker::PhantomPinned;

tokio-stream/src/stream_map.rs

@@ -605,10 +605,10 @@ mod rand {
     /// Fast random number generate
     ///
     /// Implement xorshift64+: 2 32-bit xorshift sequences added together.
-    /// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
-    /// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
+    /// Shift triplet `[17,7,16]` was calculated as indicated in Marsaglia's
+    /// Xorshift paper: <https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf>
     /// This generator passes the SmallCrush suite, part of TestU01 framework:
-    /// http://simul.iro.umontreal.ca/testu01/tu01.html
+    /// <http://simul.iro.umontreal.ca/testu01/tu01.html>
     #[derive(Debug)]
     pub(crate) struct FastRand {
         one: Cell<u32>,

tokio-stream/src/wrappers.rs

@@ -0,0 +1,71 @@
+//! Wrappers for Tokio types that implement `Stream`.
+//!
+#![cfg_attr(
+    unix,
+    doc = "You are viewing documentation built under unix. To view windows-specific wrappers, change to the `x86_64-pc-windows-msvc` platform."
+)]
+#![cfg_attr(
+    windows,
+    doc = "You are viewing documentation built under windows. To view unix-specific wrappers, change to the `x86_64-unknown-linux-gnu` platform."
+)]
+
+/// Error types for the wrappers.
+pub mod errors {
+    cfg_sync! {
+        pub use crate::wrappers::broadcast::BroadcastStreamRecvError;
+    }
+}
+
+mod mpsc_bounded;
+pub use mpsc_bounded::ReceiverStream;
+
+mod mpsc_unbounded;
+pub use mpsc_unbounded::UnboundedReceiverStream;
+
+cfg_sync! {
+    mod broadcast;
+    pub use broadcast::BroadcastStream;
+
+    mod watch;
+    pub use watch::WatchStream;
+}
+
+cfg_signal! {
+    #[cfg(unix)]
+    mod signal_unix;
+    #[cfg(unix)]
+    pub use signal_unix::SignalStream;
+
+    #[cfg(windows)]
+    mod signal_windows;
+    #[cfg(windows)]
+    pub use signal_windows::{CtrlCStream, CtrlBreakStream};
+}
+
+cfg_time! {
+    mod interval;
+    pub use interval::IntervalStream;
+}
+
+cfg_net! {
+    mod tcp_listener;
+    pub use tcp_listener::TcpListenerStream;
+
+    #[cfg(unix)]
+    mod unix_listener;
+    #[cfg(unix)]
+    pub use unix_listener::UnixListenerStream;
+}
+
+cfg_io_util! {
+    mod split;
+    pub use split::SplitStream;
+
+    mod lines;
+    pub use lines::LinesStream;
+}
+
+cfg_fs! {
+    mod read_dir;
+    pub use read_dir::ReadDirStream;
+}

tokio-stream/src/wrappers/broadcast.rs

@@ -0,0 +1,73 @@
+use std::pin::Pin;
+use tokio::sync::broadcast::error::RecvError;
+use tokio::sync::broadcast::Receiver;
+
+use futures_core::Stream;
+use tokio_util::sync::ReusableBoxFuture;
+
+use std::fmt;
+use std::task::{Context, Poll};
+
+/// A wrapper around [`tokio::sync::broadcast::Receiver`] that implements [`Stream`].
+///
+/// [`tokio::sync::broadcast::Receiver`]: struct@tokio::sync::broadcast::Receiver
+/// [`Stream`]: trait@crate::Stream
+#[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
+pub struct BroadcastStream<T> {
+    inner: ReusableBoxFuture<(Result<T, RecvError>, Receiver<T>)>,
+}
+
+/// An error returned from the inner stream of a [`BroadcastStream`].
+#[derive(Debug, PartialEq)]
+pub enum BroadcastStreamRecvError {
+    /// The receiver lagged too far behind. Attempting to receive again will
+    /// return the oldest message still retained by the channel.
+    ///
+    /// Includes the number of skipped messages.
+    Lagged(u64),
+}
+
+impl fmt::Display for BroadcastStreamRecvError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BroadcastStreamRecvError::Lagged(amt) => write!(f, "channel lagged by {}", amt),
+        }
+    }
+}
+
+impl std::error::Error for BroadcastStreamRecvError {}
+
+async fn make_future<T: Clone>(mut rx: Receiver<T>) -> (Result<T, RecvError>, Receiver<T>) {
+    let result = rx.recv().await;
+    (result, rx)
+}
+
+impl<T: 'static + Clone + Send> BroadcastStream<T> {
+    /// Create a new `BroadcastStream`.
+    pub fn new(rx: Receiver<T>) -> Self {
+        Self {
+            inner: ReusableBoxFuture::new(make_future(rx)),
+        }
+    }
+}
+
+impl<T: 'static + Clone + Send> Stream for BroadcastStream<T> {
+    type Item = Result<T, BroadcastStreamRecvError>;
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let (result, rx) = ready!(self.inner.poll(cx));
+        self.inner.set(make_future(rx));
+        match result {
+            Ok(item) => Poll::Ready(Some(Ok(item))),
+            Err(RecvError::Closed) => Poll::Ready(None),
+            Err(RecvError::Lagged(n)) => {
+                Poll::Ready(Some(Err(BroadcastStreamRecvError::Lagged(n))))
+            }
+        }
+    }
+}
+
+impl<T> fmt::Debug for BroadcastStream<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("BroadcastStream").finish()
+    }
+}

tokio-stream/src/wrappers/interval.rs

@@ -0,0 +1,50 @@
+use crate::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::time::{Instant, Interval};
+
+/// A wrapper around [`Interval`] that implements [`Stream`].
+///
+/// [`Interval`]: struct@tokio::time::Interval
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(feature = "time")))]
+pub struct IntervalStream {
+    inner: Interval,
+}
+
+impl IntervalStream {
+    /// Create a new `IntervalStream`.
+    pub fn new(interval: Interval) -> Self {
+        Self { inner: interval }
+    }
+
+    /// Get back the inner `Interval`.
+    pub fn into_inner(self) -> Interval {
+        self.inner
+    }
+}
+
+impl Stream for IntervalStream {
+    type Item = Instant;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Instant>> {
+        self.inner.poll_tick(cx).map(Some)
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (std::usize::MAX, None)
+    }
+}
+
+impl AsRef<Interval> for IntervalStream {
+    fn as_ref(&self) -> &Interval {
+        &self.inner
+    }
+}
+
+impl AsMut<Interval> for IntervalStream {
+    fn as_mut(&mut self) -> &mut Interval {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/lines.rs

@@ -0,0 +1,60 @@
+use crate::Stream;
+use pin_project_lite::pin_project;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncBufRead, Lines};
+
+pin_project! {
+    /// A wrapper around [`tokio::io::Lines`] that implements [`Stream`].
+    ///
+    /// [`tokio::io::Lines`]: struct@tokio::io::Lines
+    /// [`Stream`]: trait@crate::Stream
+    #[derive(Debug)]
+    #[cfg_attr(docsrs, doc(cfg(feature = "io-util")))]
+    pub struct LinesStream<R> {
+        #[pin]
+        inner: Lines<R>,
+    }
+}
+
+impl<R> LinesStream<R> {
+    /// Create a new `LinesStream`.
+    pub fn new(lines: Lines<R>) -> Self {
+        Self { inner: lines }
+    }
+
+    /// Get back the inner `Lines`.
+    pub fn into_inner(self) -> Lines<R> {
+        self.inner
+    }
+
+    /// Obtain a pinned reference to the inner `Lines<R>`.
+    #[allow(clippy::wrong_self_convention)] // https://github.com/rust-lang/rust-clippy/issues/4546
+    pub fn as_pin_mut(self: Pin<&mut Self>) -> Pin<&mut Lines<R>> {
+        self.project().inner
+    }
+}
+
+impl<R: AsyncBufRead> Stream for LinesStream<R> {
+    type Item = io::Result<String>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.project()
+            .inner
+            .poll_next_line(cx)
+            .map(Result::transpose)
+    }
+}
+
+impl<R> AsRef<Lines<R>> for LinesStream<R> {
+    fn as_ref(&self) -> &Lines<R> {
+        &self.inner
+    }
+}
+
+impl<R> AsMut<Lines<R>> for LinesStream<R> {
+    fn as_mut(&mut self) -> &mut Lines<R> {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/mpsc_bounded.rs

@@ -0,0 +1,59 @@
+use crate::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc::Receiver;
+
+/// A wrapper around [`tokio::sync::mpsc::Receiver`] that implements [`Stream`].
+///
+/// [`tokio::sync::mpsc::Receiver`]: struct@tokio::sync::mpsc::Receiver
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+pub struct ReceiverStream<T> {
+    inner: Receiver<T>,
+}
+
+impl<T> ReceiverStream<T> {
+    /// Create a new `ReceiverStream`.
+    pub fn new(recv: Receiver<T>) -> Self {
+        Self { inner: recv }
+    }
+
+    /// Get back the inner `Receiver`.
+    pub fn into_inner(self) -> Receiver<T> {
+        self.inner
+    }
+
+    /// Closes the receiving half of a channel without dropping it.
+    ///
+    /// This prevents any further messages from being sent on the channel while
+    /// still enabling the receiver to drain messages that are buffered. Any
+    /// outstanding [`Permit`] values will still be able to send messages.
+    ///
+    /// To guarantee no messages are dropped, after calling `close()`, you must
+    /// receive all items from the stream until `None` is returned.
+    ///
+    /// [`Permit`]: struct@tokio::sync::mpsc::Permit
+    pub fn close(&mut self) {
+        self.inner.close()
+    }
+}
+
+impl<T> Stream for ReceiverStream<T> {
+    type Item = T;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.inner.poll_recv(cx)
+    }
+}
+
+impl<T> AsRef<Receiver<T>> for ReceiverStream<T> {
+    fn as_ref(&self) -> &Receiver<T> {
+        &self.inner
+    }
+}
+
+impl<T> AsMut<Receiver<T>> for ReceiverStream<T> {
+    fn as_mut(&mut self) -> &mut Receiver<T> {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/mpsc_unbounded.rs

@@ -0,0 +1,53 @@
+use crate::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc::UnboundedReceiver;
+
+/// A wrapper around [`tokio::sync::mpsc::UnboundedReceiver`] that implements [`Stream`].
+///
+/// [`tokio::sync::mpsc::UnboundedReceiver`]: struct@tokio::sync::mpsc::UnboundedReceiver
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+pub struct UnboundedReceiverStream<T> {
+    inner: UnboundedReceiver<T>,
+}
+
+impl<T> UnboundedReceiverStream<T> {
+    /// Create a new `UnboundedReceiverStream`.
+    pub fn new(recv: UnboundedReceiver<T>) -> Self {
+        Self { inner: recv }
+    }
+
+    /// Get back the inner `UnboundedReceiver`.
+    pub fn into_inner(self) -> UnboundedReceiver<T> {
+        self.inner
+    }
+
+    /// Closes the receiving half of a channel without dropping it.
+    ///
+    /// This prevents any further messages from being sent on the channel while
+    /// still enabling the receiver to drain messages that are buffered.
+    pub fn close(&mut self) {
+        self.inner.close()
+    }
+}
+
+impl<T> Stream for UnboundedReceiverStream<T> {
+    type Item = T;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.inner.poll_recv(cx)
+    }
+}
+
+impl<T> AsRef<UnboundedReceiver<T>> for UnboundedReceiverStream<T> {
+    fn as_ref(&self) -> &UnboundedReceiver<T> {
+        &self.inner
+    }
+}
+
+impl<T> AsMut<UnboundedReceiver<T>> for UnboundedReceiverStream<T> {
+    fn as_mut(&mut self) -> &mut UnboundedReceiver<T> {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/read_dir.rs

@@ -0,0 +1,47 @@
+use crate::Stream;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::fs::{DirEntry, ReadDir};
+
+/// A wrapper around [`tokio::fs::ReadDir`] that implements [`Stream`].
+///
+/// [`tokio::fs::ReadDir`]: struct@tokio::fs::ReadDir
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
+pub struct ReadDirStream {
+    inner: ReadDir,
+}
+
+impl ReadDirStream {
+    /// Create a new `ReadDirStream`.
+    pub fn new(read_dir: ReadDir) -> Self {
+        Self { inner: read_dir }
+    }
+
+    /// Get back the inner `ReadDir`.
+    pub fn into_inner(self) -> ReadDir {
+        self.inner
+    }
+}
+
+impl Stream for ReadDirStream {
+    type Item = io::Result<DirEntry>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.inner.poll_next_entry(cx).map(Result::transpose)
+    }
+}
+
+impl AsRef<ReadDir> for ReadDirStream {
+    fn as_ref(&self) -> &ReadDir {
+        &self.inner
+    }
+}
+
+impl AsMut<ReadDir> for ReadDirStream {
+    fn as_mut(&mut self) -> &mut ReadDir {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/signal_unix.rs

@@ -0,0 +1,46 @@
+use crate::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::signal::unix::Signal;
+
+/// A wrapper around [`Signal`] that implements [`Stream`].
+///
+/// [`Signal`]: struct@tokio::signal::unix::Signal
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(all(unix, feature = "signal"))))]
+pub struct SignalStream {
+    inner: Signal,
+}
+
+impl SignalStream {
+    /// Create a new `SignalStream`.
+    pub fn new(interval: Signal) -> Self {
+        Self { inner: interval }
+    }
+
+    /// Get back the inner `Signal`.
+    pub fn into_inner(self) -> Signal {
+        self.inner
+    }
+}
+
+impl Stream for SignalStream {
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<()>> {
+        self.inner.poll_recv(cx)
+    }
+}
+
+impl AsRef<Signal> for SignalStream {
+    fn as_ref(&self) -> &Signal {
+        &self.inner
+    }
+}
+
+impl AsMut<Signal> for SignalStream {
+    fn as_mut(&mut self) -> &mut Signal {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/signal_windows.rs

@@ -0,0 +1,88 @@
+use crate::Stream;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::signal::windows::{CtrlBreak, CtrlC};
+
+/// A wrapper around [`CtrlC`] that implements [`Stream`].
+///
+/// [`CtrlC`]: struct@tokio::signal::windows::CtrlC
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))]
+pub struct CtrlCStream {
+    inner: CtrlC,
+}
+
+impl CtrlCStream {
+    /// Create a new `CtrlCStream`.
+    pub fn new(interval: CtrlC) -> Self {
+        Self { inner: interval }
+    }
+
+    /// Get back the inner `CtrlC`.
+    pub fn into_inner(self) -> CtrlC {
+        self.inner
+    }
+}
+
+impl Stream for CtrlCStream {
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<()>> {
+        self.inner.poll_recv(cx)
+    }
+}
+
+impl AsRef<CtrlC> for CtrlCStream {
+    fn as_ref(&self) -> &CtrlC {
+        &self.inner
+    }
+}
+
+impl AsMut<CtrlC> for CtrlCStream {
+    fn as_mut(&mut self) -> &mut CtrlC {
+        &mut self.inner
+    }
+}
+
+/// A wrapper around [`CtrlBreak`] that implements [`Stream`].
+///
+/// [`CtrlBreak`]: struct@tokio::signal::windows::CtrlBreak
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))]
+pub struct CtrlBreakStream {
+    inner: CtrlBreak,
+}
+
+impl CtrlBreakStream {
+    /// Create a new `CtrlBreakStream`.
+    pub fn new(interval: CtrlBreak) -> Self {
+        Self { inner: interval }
+    }
+
+    /// Get back the inner `CtrlBreak`.
+    pub fn into_inner(self) -> CtrlBreak {
+        self.inner
+    }
+}
+
+impl Stream for CtrlBreakStream {
+    type Item = ();
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<()>> {
+        self.inner.poll_recv(cx)
+    }
+}
+
+impl AsRef<CtrlBreak> for CtrlBreakStream {
+    fn as_ref(&self) -> &CtrlBreak {
+        &self.inner
+    }
+}
+
+impl AsMut<CtrlBreak> for CtrlBreakStream {
+    fn as_mut(&mut self) -> &mut CtrlBreak {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/split.rs

@@ -0,0 +1,60 @@
+use crate::Stream;
+use pin_project_lite::pin_project;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncBufRead, Split};
+
+pin_project! {
+    /// A wrapper around [`tokio::io::Split`] that implements [`Stream`].
+    ///
+    /// [`tokio::io::Split`]: struct@tokio::io::Split
+    /// [`Stream`]: trait@crate::Stream
+    #[derive(Debug)]
+    #[cfg_attr(docsrs, doc(cfg(feature = "io-util")))]
+    pub struct SplitStream<R> {
+        #[pin]
+        inner: Split<R>,
+    }
+}
+
+impl<R> SplitStream<R> {
+    /// Create a new `SplitStream`.
+    pub fn new(split: Split<R>) -> Self {
+        Self { inner: split }
+    }
+
+    /// Get back the inner `Split`.
+    pub fn into_inner(self) -> Split<R> {
+        self.inner
+    }
+
+    /// Obtain a pinned reference to the inner `Split<R>`.
+    #[allow(clippy::wrong_self_convention)] // https://github.com/rust-lang/rust-clippy/issues/4546
+    pub fn as_pin_mut(self: Pin<&mut Self>) -> Pin<&mut Split<R>> {
+        self.project().inner
+    }
+}
+
+impl<R: AsyncBufRead> Stream for SplitStream<R> {
+    type Item = io::Result<Vec<u8>>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.project()
+            .inner
+            .poll_next_segment(cx)
+            .map(Result::transpose)
+    }
+}
+
+impl<R> AsRef<Split<R>> for SplitStream<R> {
+    fn as_ref(&self) -> &Split<R> {
+        &self.inner
+    }
+}
+
+impl<R> AsMut<Split<R>> for SplitStream<R> {
+    fn as_mut(&mut self) -> &mut Split<R> {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/tcp_listener.rs

@@ -0,0 +1,54 @@
+use crate::Stream;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::net::{TcpListener, TcpStream};
+
+/// A wrapper around [`TcpListener`] that implements [`Stream`].
+///
+/// [`TcpListener`]: struct@tokio::net::TcpListener
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(feature = "net")))]
+pub struct TcpListenerStream {
+    inner: TcpListener,
+}
+
+impl TcpListenerStream {
+    /// Create a new `TcpListenerStream`.
+    pub fn new(listener: TcpListener) -> Self {
+        Self { inner: listener }
+    }
+
+    /// Get back the inner `TcpListener`.
+    pub fn into_inner(self) -> TcpListener {
+        self.inner
+    }
+}
+
+impl Stream for TcpListenerStream {
+    type Item = io::Result<TcpStream>;
+
+    fn poll_next(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<io::Result<TcpStream>>> {
+        match self.inner.poll_accept(cx) {
+            Poll::Ready(Ok((stream, _))) => Poll::Ready(Some(Ok(stream))),
+            Poll::Ready(Err(err)) => Poll::Ready(Some(Err(err))),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl AsRef<TcpListener> for TcpListenerStream {
+    fn as_ref(&self) -> &TcpListener {
+        &self.inner
+    }
+}
+
+impl AsMut<TcpListener> for TcpListenerStream {
+    fn as_mut(&mut self) -> &mut TcpListener {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/unix_listener.rs

@@ -0,0 +1,54 @@
+use crate::Stream;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::net::{UnixListener, UnixStream};
+
+/// A wrapper around [`UnixListener`] that implements [`Stream`].
+///
+/// [`UnixListener`]: struct@tokio::net::UnixListener
+/// [`Stream`]: trait@crate::Stream
+#[derive(Debug)]
+#[cfg_attr(docsrs, doc(cfg(all(unix, feature = "net"))))]
+pub struct UnixListenerStream {
+    inner: UnixListener,
+}
+
+impl UnixListenerStream {
+    /// Create a new `UnixListenerStream`.
+    pub fn new(listener: UnixListener) -> Self {
+        Self { inner: listener }
+    }
+
+    /// Get back the inner `UnixListener`.
+    pub fn into_inner(self) -> UnixListener {
+        self.inner
+    }
+}
+
+impl Stream for UnixListenerStream {
+    type Item = io::Result<UnixStream>;
+
+    fn poll_next(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<io::Result<UnixStream>>> {
+        match self.inner.poll_accept(cx) {
+            Poll::Ready(Ok((stream, _))) => Poll::Ready(Some(Ok(stream))),
+            Poll::Ready(Err(err)) => Poll::Ready(Some(Err(err))),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl AsRef<UnixListener> for UnixListenerStream {
+    fn as_ref(&self) -> &UnixListener {
+        &self.inner
+    }
+}
+
+impl AsMut<UnixListener> for UnixListenerStream {
+    fn as_mut(&mut self) -> &mut UnixListener {
+        &mut self.inner
+    }
+}

tokio-stream/src/wrappers/watch.rs

@@ -0,0 +1,96 @@
+use std::pin::Pin;
+use tokio::sync::watch::Receiver;
+
+use futures_core::Stream;
+use tokio_util::sync::ReusableBoxFuture;
+
+use std::fmt;
+use std::task::{Context, Poll};
+use tokio::sync::watch::error::RecvError;
+
+/// A wrapper around [`tokio::sync::watch::Receiver`] that implements [`Stream`].
+///
+/// This stream will always start by yielding the current value when the WatchStream is polled,
+/// regardles of whether it was the initial value or sent afterwards.
+///
+/// # Examples
+///
+/// ```
+/// # #[tokio::main]
+/// # async fn main() {
+/// use tokio_stream::{StreamExt, wrappers::WatchStream};
+/// use tokio::sync::watch;
+///
+/// let (tx, rx) = watch::channel("hello");
+/// let mut rx = WatchStream::new(rx);
+///
+/// assert_eq!(rx.next().await, Some("hello"));
+///
+/// tx.send("goodbye").unwrap();
+/// assert_eq!(rx.next().await, Some("goodbye"));
+/// # }
+/// ```
+///
+/// ```
+/// # #[tokio::main]
+/// # async fn main() {
+/// use tokio_stream::{StreamExt, wrappers::WatchStream};
+/// use tokio::sync::watch;
+///
+/// let (tx, rx) = watch::channel("hello");
+/// let mut rx = WatchStream::new(rx);
+///
+/// tx.send("goodbye").unwrap();
+/// assert_eq!(rx.next().await, Some("goodbye"));
+/// # }
+/// ```
+///
+/// [`tokio::sync::watch::Receiver`]: struct@tokio::sync::watch::Receiver
+/// [`Stream`]: trait@crate::Stream
+#[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
+pub struct WatchStream<T> {
+    inner: ReusableBoxFuture<(Result<(), RecvError>, Receiver<T>)>,
+}
+
+async fn make_future<T: Clone + Send + Sync>(
+    mut rx: Receiver<T>,
+) -> (Result<(), RecvError>, Receiver<T>) {
+    let result = rx.changed().await;
+    (result, rx)
+}
+
+impl<T: 'static + Clone + Unpin + Send + Sync> WatchStream<T> {
+    /// Create a new `WatchStream`.
+    pub fn new(rx: Receiver<T>) -> Self {
+        Self {
+            inner: ReusableBoxFuture::new(async move { (Ok(()), rx) }),
+        }
+    }
+}
+
+impl<T: Clone + 'static + Send + Sync> Stream for WatchStream<T> {
+    type Item = T;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let (result, rx) = ready!(self.inner.poll(cx));
+        match result {
+            Ok(_) => {
+                let received = (*rx.borrow()).clone();
+                self.inner.set(make_future(rx));
+                Poll::Ready(Some(received))
+            }
+            Err(_) => {
+                self.inner.set(make_future(rx));
+                Poll::Ready(None)
+            }
+        }
+    }
+}
+
+impl<T> Unpin for WatchStream<T> {}
+
+impl<T> fmt::Debug for WatchStream<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("WatchStream").finish()
+    }
+}

tokio-stream/tests/async_send_sync.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::diverging_sub_expression)]
+
 use std::rc::Rc;
 
 #[allow(dead_code)]

tokio-stream/tests/stream_chain.rs

@@ -89,12 +89,12 @@ fn size_overflow() {
         }
 
         fn size_hint(&self) -> (usize, Option<usize>) {
-            (usize::max_value(), Some(usize::max_value()))
+            (usize::MAX, Some(usize::MAX))
         }
     }
 
     let m1 = Monster;
     let m2 = Monster;
     let m = m1.chain(m2);
-    assert_eq!(m.size_hint(), (usize::max_value(), None));
+    assert_eq!(m.size_hint(), (usize::MAX, None));
 }

tokio-stream/tests/stream_merge.rs

@@ -72,12 +72,12 @@ fn size_overflow() {
         }
 
         fn size_hint(&self) -> (usize, Option<usize>) {
-            (usize::max_value(), Some(usize::max_value()))
+            (usize::MAX, Some(usize::MAX))
         }
     }
 
     let m1 = Monster;
     let m2 = Monster;
     let m = m1.merge(m2);
-    assert_eq!(m.size_hint(), (usize::max_value(), None));
+    assert_eq!(m.size_hint(), (usize::MAX, None));
 }

tokio-test/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.4.2 (May 14, 2021)
+
+- test: add `assert_elapsed!` macro ([#3728])
+
+[#3728]: https://github.com/tokio-rs/tokio/pull/3728
+
+# 0.4.1 (March 10, 2021)
+
+- Fix `io::Mock` to be `Send` and `Sync` ([#3594])
+
+[#3594]: https://github.com/tokio-rs/tokio/pull/3594
+
 # 0.4.0 (December 23, 2020)
 
 - Track `tokio` 1.0 release.

tokio-test/Cargo.toml

@@ -2,25 +2,24 @@
 name = "tokio-test"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update html_root_url.
 # - Update doc url
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-test-0.4.x" git tag.
-version = "0.4.0"
+version = "0.4.2"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-test/0.4.0/tokio_test"
+documentation = "https://docs.rs/tokio-test/0.4.2/tokio_test"
 description = """
 Testing utilities for Tokio- and futures-based code
 """
 categories = ["asynchronous", "testing"]
 
 [dependencies]
-tokio = { version = "1.0.0", path = "../tokio", features = ["rt", "sync", "time", "test-util"] }
+tokio = { version = "1.2.0", path = "../tokio", features = ["rt", "sync", "time", "test-util"] }
 tokio-stream = { version = "0.1", path = "../tokio-stream" }
 async-stream = "0.3"
 
@@ -28,7 +27,7 @@ bytes = "1.0.0"
 futures-core = "0.3.0"
 
 [dev-dependencies]
-tokio = { version = "1.0.0", path = "../tokio", features = ["full"] }
+tokio = { version = "1.2.0", path = "../tokio", features = ["full"] }
 futures-util = "0.3.0"
 
 [package.metadata.docs.rs]

tokio-test/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Tokio Contributors
+Copyright (c) 2021 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-test/src/io.rs

@@ -21,6 +21,7 @@
 use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 use tokio::sync::mpsc;
 use tokio::time::{self, Duration, Instant, Sleep};
+use tokio_stream::wrappers::UnboundedReceiverStream;
 
 use futures_core::{ready, Stream};
 use std::collections::VecDeque;
@@ -69,8 +70,7 @@ struct Inner {
     waiting: Option<Instant>,
     sleep: Option<Pin<Box<Sleep>>>,
     read_wait: Option<Waker>,
-    // rx: mpsc::UnboundedReceiver<Action>,
-    rx: Pin<Box<dyn Stream<Item = Action> + Send>>,
+    rx: UnboundedReceiverStream<Action>,
 }
 
 impl Builder {
@@ -185,13 +185,9 @@ impl Handle {
 
 impl Inner {
     fn new(actions: VecDeque<Action>) -> (Inner, Handle) {
-        let (tx, mut rx) = mpsc::unbounded_channel();
+        let (tx, rx) = mpsc::unbounded_channel();
 
-        let rx = Box::pin(async_stream::stream! {
-            while let Some(item) = rx.recv().await {
-                yield item;
-            }
-        });
+        let rx = UnboundedReceiverStream::new(rx);
 
         let inner = Inner {
             actions,

tokio-test/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc(html_root_url = "https://docs.rs/tokio-test/0.4.0")]
 #![warn(
     missing_debug_implementations,
     missing_docs,
@@ -22,9 +21,9 @@ pub mod task;
 /// future completes.
 ///
 /// For more information, see the documentation for
-/// [`tokio::runtime::current_thread::Runtime::block_on`][runtime-block-on].
+/// [`tokio::runtime::Runtime::block_on`][runtime-block-on].
 ///
-/// [runtime-block-on]: https://docs.rs/tokio/0.2.0-alpha.2/tokio/runtime/current_thread/struct.Runtime.html#method.block_on
+/// [runtime-block-on]: https://docs.rs/tokio/1.3.0/tokio/runtime/struct.Runtime.html#method.block_on
 pub fn block_on<F: std::future::Future>(future: F) -> F::Output {
     use tokio::runtime;
 

tokio-test/src/macros.rs

@@ -259,3 +259,37 @@ macro_rules! assert_err {
         }
     }};
 }
+
+/// Asserts that an exact duration has elapsed since since the start instant ±1ms.
+///
+/// ```rust
+/// use tokio::time::{self, Instant};
+/// use std::time::Duration;
+/// use tokio_test::assert_elapsed;
+/// # async fn test_time_passed() {
+///
+/// let start = Instant::now();
+/// let dur = Duration::from_millis(50);
+/// time::sleep(dur).await;
+/// assert_elapsed!(start, dur);
+/// # }
+/// ```
+///
+/// This 1ms buffer is required because Tokio's hashed-wheel timer has finite time resolution and
+/// will not always sleep for the exact interval.
+#[macro_export]
+macro_rules! assert_elapsed {
+    ($start:expr, $dur:expr) => {{
+        let elapsed = $start.elapsed();
+        // type ascription improves compiler error when wrong type is passed
+        let lower: std::time::Duration = $dur;
+
+        // Handles ms rounding
+        assert!(
+            elapsed >= lower && elapsed <= lower + std::time::Duration::from_millis(1),
+            "actual = {:?}, expected = {:?}",
+            elapsed,
+            lower
+        );
+    }};
+}

tokio-util/CHANGELOG.md

@@ -1,3 +1,84 @@
+# 0.6.7 (May 14, 2021)
+
+### Added
+
+- udp: make `UdpFramed` take `Borrow<UdpSocket>` ([#3451])
+- compat: implement `AsRawFd`/`AsRawHandle` for `Compat<T>` ([#3765])
+
+[#3451]: https://github.com/tokio-rs/tokio/pull/3451
+[#3765]: https://github.com/tokio-rs/tokio/pull/3765
+
+# 0.6.6 (April 12, 2021)
+
+### Added
+
+- util: makes `Framed` and `FramedStream` resumable after eof ([#3272])
+- util: add `PollSemaphore::{add_permits, available_permits}` ([#3683])
+
+### Fixed
+
+- chore: avoid allocation if `PollSemaphore` is unused ([#3634])
+
+[#3272]: https://github.com/tokio-rs/tokio/pull/3272
+[#3634]: https://github.com/tokio-rs/tokio/pull/3634
+[#3683]: https://github.com/tokio-rs/tokio/pull/3683
+
+# 0.6.5 (March 20, 2021)
+
+### Fixed
+
+- util: annotate time module as requiring `time` feature ([#3606])
+
+[#3606]: https://github.com/tokio-rs/tokio/pull/3606
+
+# 0.6.4 (March 9, 2021)
+
+### Added
+
+- codec: `AnyDelimiter` codec ([#3406])
+- sync: add pollable `mpsc::Sender` ([#3490])
+
+### Fixed
+
+- codec: `LinesCodec` should only return `MaxLineLengthExceeded` once per line ([#3556])
+- sync: fuse PollSemaphore ([#3578])
+
+[#3406]: https://github.com/tokio-rs/tokio/pull/3406
+[#3490]: https://github.com/tokio-rs/tokio/pull/3490
+[#3556]: https://github.com/tokio-rs/tokio/pull/3556
+[#3578]: https://github.com/tokio-rs/tokio/pull/3578
+
+# 0.6.3 (January 31, 2021)
+
+### Added
+
+- sync: add `ReusableBoxFuture` utility ([#3464])
+
+### Changed
+
+- sync: use `ReusableBoxFuture` for `PollSemaphore` ([#3463])
+- deps: remove `async-stream` dependency ([#3463])
+- deps: remove `tokio-stream` dependency ([#3487])
+
+# 0.6.2 (January 21, 2021)
+
+### Added
+
+- sync: add pollable `Semaphore` ([#3444])
+
+### Fixed
+
+- time: fix panics on updating `DelayQueue` entries ([#3270])
+
+# 0.6.1 (January 12, 2021)
+
+### Added
+
+- codec: `get_ref()`, `get_mut()`, `get_pin_mut()` and `into_inner()` for
+  `Framed`, `FramedRead`, `FramedWrite` and `StreamReader` ([#3364]).
+- codec: `write_buffer()` and `write_buffer_mut()` for `Framed` and
+  `FramedWrite` ([#3387]).
+
 # 0.6.0 (December 23, 2020)
 
 ### Changed
@@ -53,6 +134,13 @@
 
 - Initial release
 
+[#3487]: https://github.com/tokio-rs/tokio/pull/3487
+[#3464]: https://github.com/tokio-rs/tokio/pull/3464
+[#3463]: https://github.com/tokio-rs/tokio/pull/3463
+[#3444]: https://github.com/tokio-rs/tokio/pull/3444
+[#3387]: https://github.com/tokio-rs/tokio/pull/3387
+[#3364]: https://github.com/tokio-rs/tokio/pull/3364
+[#3270]: https://github.com/tokio-rs/tokio/pull/3270
 [#2326]: https://github.com/tokio-rs/tokio/pull/2326
 [#2215]: https://github.com/tokio-rs/tokio/pull/2215
 [#2198]: https://github.com/tokio-rs/tokio/pull/2198

tokio-util/Cargo.toml

@@ -2,18 +2,17 @@
 name = "tokio-util"
 # When releasing to crates.io:
 # - Remove path dependencies
-# - Update html_root_url.
 # - Update doc url
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "tokio-util-0.6.x" git tag.
-version = "0.6.0"
+version = "0.6.7"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-util/0.6.0/tokio_util"
+documentation = "https://docs.rs/tokio-util/0.6.7/tokio_util"
 description = """
 Additional utilities for working with Tokio.
 """
@@ -36,8 +35,7 @@ rt = ["tokio/rt"]
 __docs_rs = ["futures-util"]
 
 [dependencies]
-tokio = { version = "1.0.0", path = "../tokio" }
-tokio-stream = { version = "0.1", path = "../tokio-stream" }
+tokio = { version = "1.0.0", path = "../tokio", features = ["sync"] }
 
 bytes = "1.0.0"
 futures-core = "0.3.0"
@@ -51,11 +49,11 @@ slab = { version = "0.4.1", optional = true } # Backs `DelayQueue`
 [dev-dependencies]
 tokio = { version = "1.0.0", path = "../tokio", features = ["full"] }
 tokio-test = { version = "0.4.0", path = "../tokio-test" }
-
-futures = "0.3.0"
-futures-test = "0.3.5"
+tokio-stream = { version = "0.1", path = "../tokio-stream" }
 
 async-stream = "0.3.0"
+futures = "0.3.0"
+futures-test = "0.3.5"
 
 [package.metadata.docs.rs]
 all-features = true

tokio-util/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2020 Tokio Contributors
+Copyright (c) 2021 Tokio Contributors
 
 Permission is hereby granted, free of charge, to any
 person obtaining a copy of this software and associated

tokio-util/src/cfg.rs

@@ -47,3 +47,13 @@ macro_rules! cfg_rt {
         )*
     }
 }
+
+macro_rules! cfg_time {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "time")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "time")))]
+            $item
+        )*
+    }
+}

tokio-util/src/codec/any_delimiter_codec.rs

@@ -0,0 +1,263 @@
+use crate::codec::decoder::Decoder;
+use crate::codec::encoder::Encoder;
+
+use bytes::{Buf, BufMut, Bytes, BytesMut};
+use std::{cmp, fmt, io, str, usize};
+
+const DEFAULT_SEEK_DELIMITERS: &[u8] = b",;\n\r";
+const DEFAULT_SEQUENCE_WRITER: &[u8] = b",";
+/// A simple [`Decoder`] and [`Encoder`] implementation that splits up data into chunks based on any character in the given delimiter string.
+///
+/// [`Decoder`]: crate::codec::Decoder
+/// [`Encoder`]: crate::codec::Encoder
+///
+/// # Example
+/// Decode string of bytes containing various different delimiters.
+///
+/// [`BytesMut`]: bytes::BytesMut
+/// [`Error`]: std::io::Error
+///
+/// ```
+/// use tokio_util::codec::{AnyDelimiterCodec, Decoder};
+/// use bytes::{BufMut, BytesMut};
+///
+/// #
+/// # #[tokio::main(flavor = "current_thread")]
+/// # async fn main() -> Result<(), std::io::Error> {
+/// let mut codec = AnyDelimiterCodec::new(b",;\r\n".to_vec(),b";".to_vec());
+/// let buf = &mut BytesMut::new();
+/// buf.reserve(200);
+/// buf.put_slice(b"chunk 1,chunk 2;chunk 3\n\r");
+/// assert_eq!("chunk 1", codec.decode(buf).unwrap().unwrap());
+/// assert_eq!("chunk 2", codec.decode(buf).unwrap().unwrap());
+/// assert_eq!("chunk 3", codec.decode(buf).unwrap().unwrap());
+/// assert_eq!("", codec.decode(buf).unwrap().unwrap());
+/// assert_eq!(None, codec.decode(buf).unwrap());
+/// # Ok(())
+/// # }
+/// ```
+///
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub struct AnyDelimiterCodec {
+    // Stored index of the next index to examine for the delimiter character.
+    // This is used to optimize searching.
+    // For example, if `decode` was called with `abc` and the delimiter is '{}', it would hold `3`,
+    // because that is the next index to examine.
+    // The next time `decode` is called with `abcde}`, the method will
+    // only look at `de}` before returning.
+    next_index: usize,
+
+    /// The maximum length for a given chunk. If `usize::MAX`, chunks will be
+    /// read until a delimiter character is reached.
+    max_length: usize,
+
+    /// Are we currently discarding the remainder of a chunk which was over
+    /// the length limit?
+    is_discarding: bool,
+
+    /// The bytes that are using for search during decode
+    seek_delimiters: Vec<u8>,
+
+    /// The bytes that are using for encoding
+    sequence_writer: Vec<u8>,
+}
+
+impl AnyDelimiterCodec {
+    /// Returns a `AnyDelimiterCodec` for splitting up data into chunks.
+    ///
+    /// # Note
+    ///
+    /// The returned `AnyDelimiterCodec` will not have an upper bound on the length
+    /// of a buffered chunk. See the documentation for [`new_with_max_length`]
+    /// for information on why this could be a potential security risk.
+    ///
+    /// [`new_with_max_length`]: crate::codec::AnyDelimiterCodec::new_with_max_length()
+    pub fn new(seek_delimiters: Vec<u8>, sequence_writer: Vec<u8>) -> AnyDelimiterCodec {
+        AnyDelimiterCodec {
+            next_index: 0,
+            max_length: usize::MAX,
+            is_discarding: false,
+            seek_delimiters,
+            sequence_writer,
+        }
+    }
+
+    /// Returns a `AnyDelimiterCodec` with a maximum chunk length limit.
+    ///
+    /// If this is set, calls to `AnyDelimiterCodec::decode` will return a
+    /// [`AnyDelimiterCodecError`] when a chunk exceeds the length limit. Subsequent calls
+    /// will discard up to `limit` bytes from that chunk until a delimiter
+    /// character is reached, returning `None` until the delimiter over the limit
+    /// has been fully discarded. After that point, calls to `decode` will
+    /// function as normal.
+    ///
+    /// # Note
+    ///
+    /// Setting a length limit is highly recommended for any `AnyDelimiterCodec` which
+    /// will be exposed to untrusted input. Otherwise, the size of the buffer
+    /// that holds the chunk currently being read is unbounded. An attacker could
+    /// exploit this unbounded buffer by sending an unbounded amount of input
+    /// without any delimiter characters, causing unbounded memory consumption.
+    ///
+    /// [`AnyDelimiterCodecError`]: crate::codec::AnyDelimiterCodecError
+    pub fn new_with_max_length(
+        seek_delimiters: Vec<u8>,
+        sequence_writer: Vec<u8>,
+        max_length: usize,
+    ) -> Self {
+        AnyDelimiterCodec {
+            max_length,
+            ..AnyDelimiterCodec::new(seek_delimiters, sequence_writer)
+        }
+    }
+
+    /// Returns the maximum chunk length when decoding.
+    ///
+    /// ```
+    /// use std::usize;
+    /// use tokio_util::codec::AnyDelimiterCodec;
+    ///
+    /// let codec = AnyDelimiterCodec::new(b",;\n".to_vec(), b";".to_vec());
+    /// assert_eq!(codec.max_length(), usize::MAX);
+    /// ```
+    /// ```
+    /// use tokio_util::codec::AnyDelimiterCodec;
+    ///
+    /// let codec = AnyDelimiterCodec::new_with_max_length(b",;\n".to_vec(), b";".to_vec(), 256);
+    /// assert_eq!(codec.max_length(), 256);
+    /// ```
+    pub fn max_length(&self) -> usize {
+        self.max_length
+    }
+}
+
+impl Decoder for AnyDelimiterCodec {
+    type Item = Bytes;
+    type Error = AnyDelimiterCodecError;
+
+    fn decode(&mut self, buf: &mut BytesMut) -> Result<Option<Bytes>, AnyDelimiterCodecError> {
+        loop {
+            // Determine how far into the buffer we'll search for a delimiter. If
+            // there's no max_length set, we'll read to the end of the buffer.
+            let read_to = cmp::min(self.max_length.saturating_add(1), buf.len());
+
+            let new_chunk_offset = buf[self.next_index..read_to].iter().position(|b| {
+                self.seek_delimiters
+                    .iter()
+                    .any(|delimiter| *b == *delimiter)
+            });
+
+            match (self.is_discarding, new_chunk_offset) {
+                (true, Some(offset)) => {
+                    // If we found a new chunk, discard up to that offset and
+                    // then stop discarding. On the next iteration, we'll try
+                    // to read a chunk normally.
+                    buf.advance(offset + self.next_index + 1);
+                    self.is_discarding = false;
+                    self.next_index = 0;
+                }
+                (true, None) => {
+                    // Otherwise, we didn't find a new chunk, so we'll discard
+                    // everything we read. On the next iteration, we'll continue
+                    // discarding up to max_len bytes unless we find a new chunk.
+                    buf.advance(read_to);
+                    self.next_index = 0;
+                    if buf.is_empty() {
+                        return Ok(None);
+                    }
+                }
+                (false, Some(offset)) => {
+                    // Found a chunk!
+                    let new_chunk_index = offset + self.next_index;
+                    self.next_index = 0;
+                    let mut chunk = buf.split_to(new_chunk_index + 1);
+                    chunk.truncate(chunk.len() - 1);
+                    let chunk = chunk.freeze();
+                    return Ok(Some(chunk));
+                }
+                (false, None) if buf.len() > self.max_length => {
+                    // Reached the maximum length without finding a
+                    // new chunk, return an error and start discarding on the
+                    // next call.
+                    self.is_discarding = true;
+                    return Err(AnyDelimiterCodecError::MaxChunkLengthExceeded);
+                }
+                (false, None) => {
+                    // We didn't find a chunk or reach the length limit, so the next
+                    // call will resume searching at the current offset.
+                    self.next_index = read_to;
+                    return Ok(None);
+                }
+            }
+        }
+    }
+
+    fn decode_eof(&mut self, buf: &mut BytesMut) -> Result<Option<Bytes>, AnyDelimiterCodecError> {
+        Ok(match self.decode(buf)? {
+            Some(frame) => Some(frame),
+            None => {
+                // return remaining data, if any
+                if buf.is_empty() {
+                    None
+                } else {
+                    let chunk = buf.split_to(buf.len());
+                    self.next_index = 0;
+                    Some(chunk.freeze())
+                }
+            }
+        })
+    }
+}
+
+impl<T> Encoder<T> for AnyDelimiterCodec
+where
+    T: AsRef<str>,
+{
+    type Error = AnyDelimiterCodecError;
+
+    fn encode(&mut self, chunk: T, buf: &mut BytesMut) -> Result<(), AnyDelimiterCodecError> {
+        let chunk = chunk.as_ref();
+        buf.reserve(chunk.len() + 1);
+        buf.put(chunk.as_bytes());
+        buf.put(self.sequence_writer.as_ref());
+
+        Ok(())
+    }
+}
+
+impl Default for AnyDelimiterCodec {
+    fn default() -> Self {
+        Self::new(
+            DEFAULT_SEEK_DELIMITERS.to_vec(),
+            DEFAULT_SEQUENCE_WRITER.to_vec(),
+        )
+    }
+}
+
+/// An error occured while encoding or decoding a chunk.
+#[derive(Debug)]
+pub enum AnyDelimiterCodecError {
+    /// The maximum chunk length was exceeded.
+    MaxChunkLengthExceeded,
+    /// An IO error occurred.
+    Io(io::Error),
+}
+
+impl fmt::Display for AnyDelimiterCodecError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            AnyDelimiterCodecError::MaxChunkLengthExceeded => {
+                write!(f, "max chunk length exceeded")
+            }
+            AnyDelimiterCodecError::Io(e) => write!(f, "{}", e),
+        }
+    }
+}
+
+impl From<io::Error> for AnyDelimiterCodecError {
+    fn from(e: io::Error) -> AnyDelimiterCodecError {
+        AnyDelimiterCodecError::Io(e)
+    }
+}
+
+impl std::error::Error for AnyDelimiterCodecError {}

tokio-util/src/codec/decoder.rs

@@ -16,6 +16,20 @@ use std::io;
 /// implementing stateful streaming parsers. In many cases, though, this type
 /// will simply be a unit struct (e.g. `struct HttpDecoder`).
 ///
+/// For some underlying data-sources, namely files and FIFOs,
+/// it's possible to temporarily read 0 bytes by reaching EOF.
+///
+/// In these cases `decode_eof` will be called until it signals
+/// fullfillment of all closing frames by returning `Ok(None)`.
+/// After that, repeated attempts to read from the [`Framed`] or [`FramedRead`]
+/// will not invoke `decode` or `decode_eof` again, until data can be read
+/// during a retry.
+///
+/// It is up to the Decoder to keep track of a restart after an EOF,
+/// and to decide how to handle such an event by, for example,
+/// allowing frames to cross EOF boundaries, re-emitting opening frames, or
+/// reseting the entire internal state.
+///
 /// [`Framed`]: crate::codec::Framed
 /// [`FramedRead`]: crate::codec::FramedRead
 pub trait Decoder {
@@ -115,13 +129,18 @@ pub trait Decoder {
     /// This method defaults to calling `decode` and returns an error if
     /// `Ok(None)` is returned while there is unconsumed data in `buf`.
     /// Typically this doesn't need to be implemented unless the framing
-    /// protocol differs near the end of the stream.
+    /// protocol differs near the end of the stream, or if you need to construct
+    /// frames _across_ eof boundaries on sources that can be resumed.
     ///
     /// Note that the `buf` argument may be empty. If a previous call to
     /// `decode_eof` consumed all the bytes in the buffer, `decode_eof` will be
     /// called again until it returns `None`, indicating that there are no more
     /// frames to yield. This behavior enables returning finalization frames
     /// that may not be based on inbound data.
+    ///
+    /// Once `None` has been returned, `decode_eof` won't be called again until
+    /// an attempt to resume the stream has been made, where the underlying stream
+    /// actually returned more data.
     fn decode_eof(&mut self, buf: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self.decode(buf)? {
             Some(frame) => Ok(Some(frame)),
@@ -153,7 +172,7 @@ pub trait Decoder {
     /// calling `split` on the [`Framed`] returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`Sink`]: futures_sink::Sink
     /// [`Framed`]: crate::codec::Framed
     fn framed<T: AsyncRead + AsyncWrite + Sized>(self, io: T) -> Framed<T, Self>

tokio-util/src/codec/framed.rs

@@ -2,8 +2,8 @@ use crate::codec::decoder::Decoder;
 use crate::codec::encoder::Encoder;
 use crate::codec::framed_impl::{FramedImpl, RWFrames, ReadFrame, WriteFrame};
 
+use futures_core::Stream;
 use tokio::io::{AsyncRead, AsyncWrite};
-use tokio_stream::Stream;
 
 use bytes::BytesMut;
 use futures_sink::Sink;
@@ -20,7 +20,7 @@ pin_project! {
     /// You can create a `Framed` instance by using the [`Decoder::framed`] adapter, or
     /// by using the `new` function seen below.
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`Sink`]: futures_sink::Sink
     /// [`AsyncRead`]: tokio::io::AsyncRead
     /// [`Decoder::framed`]: crate::codec::Decoder::framed()
@@ -52,7 +52,12 @@ where
     /// calling [`split`] on the `Framed` returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// Note that, for some byte sources, the stream can be resumed after an EOF
+    /// by reading from it, even after it has returned `None`. Repeated attempts
+    /// to do so, without new data available, continue to return `None` without
+    /// creating more (closing) frames.
+    ///
+    /// [`Stream`]: futures_core::Stream
     /// [`Sink`]: futures_sink::Sink
     /// [`Decode`]: crate::codec::Decoder
     /// [`Encoder`]: crate::codec::Encoder
@@ -86,7 +91,7 @@ where
     /// calling [`split`] on the `Framed` returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`Sink`]: futures_sink::Sink
     /// [`Decode`]: crate::codec::Decoder
     /// [`Encoder`]: crate::codec::Encoder
@@ -131,7 +136,7 @@ impl<T, U> Framed<T, U> {
     /// calling [`split`] on the `Framed` returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`Sink`]: futures_sink::Sink
     /// [`Decoder`]: crate::codec::Decoder
     /// [`Encoder`]: crate::codec::Encoder
@@ -170,6 +175,16 @@ impl<T, U> Framed<T, U> {
         &mut self.inner.inner
     }
 
+    /// Returns a pinned mutable reference to the underlying I/O stream wrapped by
+    /// `Framed`.
+    ///
+    /// Note that care should be taken to not tamper with the underlying stream
+    /// of data coming in as it may corrupt the stream of frames otherwise
+    /// being worked with.
+    pub fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T> {
+        self.project().inner.project().inner
+    }
+
     /// Returns a reference to the underlying codec wrapped by
     /// `Framed`.
     ///
@@ -198,6 +213,16 @@ impl<T, U> Framed<T, U> {
         &mut self.inner.state.read.buffer
     }
 
+    /// Returns a reference to the write buffer.
+    pub fn write_buffer(&self) -> &BytesMut {
+        &self.inner.state.write.buffer
+    }
+
+    /// Returns a mutable reference to the write buffer.
+    pub fn write_buffer_mut(&mut self) -> &mut BytesMut {
+        &mut self.inner.state.write.buffer
+    }
+
     /// Consumes the `Framed`, returning its underlying I/O stream.
     ///
     /// Note that care should be taken to not tamper with the underlying stream

tokio-util/src/codec/framed_impl.rs

@@ -1,8 +1,8 @@
 use crate::codec::decoder::Decoder;
 use crate::codec::encoder::Encoder;
 
+use futures_core::Stream;
 use tokio::io::{AsyncRead, AsyncWrite};
-use tokio_stream::Stream;
 
 use bytes::BytesMut;
 use futures_core::ready;
@@ -120,42 +120,97 @@ where
 
         let mut pinned = self.project();
         let state: &mut ReadFrame = pinned.state.borrow_mut();
+        // The following loops implements a state machine with each state corresponding
+        // to a combination of the `is_readable` and `eof` flags. States persist across
+        // loop entries and most state transitions occur with a return.
+        //
+        // The intitial state is `reading`.
+        //
+        // | state   | eof   | is_readable |
+        // |---------|-------|-------------|
+        // | reading | false | false       |
+        // | framing | false | true        |
+        // | pausing | true  | true        |
+        // | paused  | true  | false       |
+        //
+        //                                 `decode_eof`
+        //                                 returns `Some`             read 0 bytes
+        //                                   │       │                 │      │
+        //                                   │       ▼                 │      ▼
+        //                                   ┌───────┐ `decode_eof`    ┌──────┐
+        //                 ┌──read 0 bytes──▶│pausing│─returns `None`─▶│paused│──┐
+        //                 │                 └───────┘                 └──────┘  │
+        // pending read┐   │                     ┌──────┐                  │  ▲  │
+        //      │      │   │                     │      │                  │  │  │
+        //      │      ▼   │                     │  `decode` returns `Some`│ pending read
+        //      │  ╔═══════╗                 ┌───────┐◀─┘                  │
+        //      └──║reading║─read n>0 bytes─▶│framing│                     │
+        //         ╚═══════╝                 └───────┘◀──────read n>0 bytes┘
+        //             ▲                         │
+        //             │                         │
+        //             └─`decode` returns `None`─┘
         loop {
-            // Repeatedly call `decode` or `decode_eof` as long as it is
-            // "readable". Readable is defined as not having returned `None`. If
-            // the upstream has returned EOF, and the decoder is no longer
-            // readable, it can be assumed that the decoder will never become
-            // readable again, at which point the stream is terminated.
+            // Repeatedly call `decode` or `decode_eof` while the buffer is "readable",
+            // i.e. it _might_ contain data consumable as a frame or closing frame.
+            // Both signal that there is no such data by returning `None`.
+            //
+            // If `decode` couldn't read a frame and the upstream source has returned eof,
+            // `decode_eof` will attemp to decode the remaining bytes as closing frames.
+            //
+            // If the underlying AsyncRead is resumable, we may continue after an EOF,
+            // but must finish emmiting all of it's associated `decode_eof` frames.
+            // Furthermore, we don't want to emit any `decode_eof` frames on retried
+            // reads after an EOF unless we've actually read more data.
             if state.is_readable {
+                // pausing or framing
                 if state.eof {
+                    // pausing
                     let frame = pinned.codec.decode_eof(&mut state.buffer)?;
+                    if frame.is_none() {
+                        state.is_readable = false; // prepare pausing -> paused
+                    }
+                    // implicit pausing -> pausing or pausing -> paused
                     return Poll::Ready(frame.map(Ok));
                 }
 
+                // framing
                 trace!("attempting to decode a frame");
 
                 if let Some(frame) = pinned.codec.decode(&mut state.buffer)? {
                     trace!("frame decoded from buffer");
+                    // implicit framing -> framing
                     return Poll::Ready(Some(Ok(frame)));
                 }
 
+                // framing -> reading
                 state.is_readable = false;
             }
-
-            assert!(!state.eof);
-
-            // Otherwise, try to read more data and try again. Make sure we've
-            // got room for at least one byte to read to ensure that we don't
-            // get a spurious 0 that looks like EOF
+            // reading or paused
+            // If we can't build a frame yet, try to read more data and try again.
+            // Make sure we've got room for at least one byte to read to ensure
+            // that we don't get a spurious 0 that looks like EOF.
             state.buffer.reserve(1);
             let bytect = match poll_read_buf(pinned.inner.as_mut(), cx, &mut state.buffer)? {
                 Poll::Ready(ct) => ct,
+                // implicit reading -> reading or implicit paused -> paused
                 Poll::Pending => return Poll::Pending,
             };
             if bytect == 0 {
+                if state.eof {
+                    // We're already at an EOF, and since we've reached this path
+                    // we're also not readable. This implies that we've already finished
+                    // our `decode_eof` handling, so we can simply return `None`.
+                    // implicit paused -> paused
+                    return Poll::Ready(None);
+                }
+                // prepare reading -> paused
                 state.eof = true;
+            } else {
+                // prepare paused -> framing or noop reading -> framing
+                state.eof = false;
             }
 
+            // paused -> framing or reading -> framing or reading -> pausing
             state.is_readable = true;
         }
     }

tokio-util/src/codec/framed_read.rs

@@ -1,8 +1,8 @@
 use crate::codec::framed_impl::{FramedImpl, ReadFrame};
 use crate::codec::Decoder;
 
+use futures_core::Stream;
 use tokio::io::AsyncRead;
-use tokio_stream::Stream;
 
 use bytes::BytesMut;
 use futures_sink::Sink;
@@ -14,7 +14,7 @@ use std::task::{Context, Poll};
 pin_project! {
     /// A [`Stream`] of messages decoded from an [`AsyncRead`].
     ///
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`AsyncRead`]: tokio::io::AsyncRead
     pub struct FramedRead<T, D> {
         #[pin]
@@ -78,6 +78,16 @@ impl<T, D> FramedRead<T, D> {
         &mut self.inner.inner
     }
 
+    /// Returns a pinned mutable reference to the underlying I/O stream wrapped by
+    /// `FramedRead`.
+    ///
+    /// Note that care should be taken to not tamper with the underlying stream
+    /// of data coming in as it may corrupt the stream of frames otherwise
+    /// being worked with.
+    pub fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T> {
+        self.project().inner.project().inner
+    }
+
     /// Consumes the `FramedRead`, returning its underlying I/O stream.
     ///
     /// Note that care should be taken to not tamper with the underlying stream

tokio-util/src/codec/framed_write.rs

@@ -1,9 +1,10 @@
 use crate::codec::encoder::Encoder;
 use crate::codec::framed_impl::{FramedImpl, WriteFrame};
 
+use futures_core::Stream;
 use tokio::io::AsyncWrite;
-use tokio_stream::Stream;
 
+use bytes::BytesMut;
 use futures_sink::Sink;
 use pin_project_lite::pin_project;
 use std::fmt;
@@ -58,6 +59,16 @@ impl<T, E> FramedWrite<T, E> {
         &mut self.inner.inner
     }
 
+    /// Returns a pinned mutable reference to the underlying I/O stream wrapped by
+    /// `FramedWrite`.
+    ///
+    /// Note that care should be taken to not tamper with the underlying stream
+    /// of data coming in as it may corrupt the stream of frames otherwise
+    /// being worked with.
+    pub fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T> {
+        self.project().inner.project().inner
+    }
+
     /// Consumes the `FramedWrite`, returning its underlying I/O stream.
     ///
     /// Note that care should be taken to not tamper with the underlying stream
@@ -76,6 +87,16 @@ impl<T, E> FramedWrite<T, E> {
     pub fn encoder_mut(&mut self) -> &mut E {
         &mut self.inner.codec
     }
+
+    /// Returns a reference to the write buffer.
+    pub fn write_buffer(&self) -> &BytesMut {
+        &self.inner.state.buffer
+    }
+
+    /// Returns a mutable reference to the write buffer.
+    pub fn write_buffer_mut(&mut self) -> &mut BytesMut {
+        &mut self.inner.state.buffer
+    }
 }
 
 // This impl just defers to the underlying FramedImpl

tokio-util/src/codec/length_delimited.rs

@@ -535,14 +535,14 @@ impl LengthDelimitedCodec {
         Ok(Some(n))
     }
 
-    fn decode_data(&self, n: usize, src: &mut BytesMut) -> io::Result<Option<BytesMut>> {
+    fn decode_data(&self, n: usize, src: &mut BytesMut) -> Option<BytesMut> {
         // At this point, the buffer has already had the required capacity
         // reserved. All there is to do is read.
         if src.len() < n {
-            return Ok(None);
+            return None;
         }
 
-        Ok(Some(src.split_to(n)))
+        Some(src.split_to(n))
     }
 }
 
@@ -562,7 +562,7 @@ impl Decoder for LengthDelimitedCodec {
             DecodeState::Data(n) => n,
         };
 
-        match self.decode_data(n, src)? {
+        match self.decode_data(n, src) {
             Some(data) => {
                 // Update the decode state
                 self.state = DecodeState::Head;

tokio-util/src/codec/lines_codec.rs

@@ -133,7 +133,7 @@ impl Decoder for LinesCodec {
                     buf.advance(read_to);
                     self.next_index = 0;
                     if buf.is_empty() {
-                        return Err(LinesCodecError::MaxLineLengthExceeded);
+                        return Ok(None);
                     }
                 }
                 (false, Some(offset)) => {

tokio-util/src/codec/mod.rs

@@ -246,7 +246,7 @@
 //!
 //! [`AsyncRead`]: tokio::io::AsyncRead
 //! [`AsyncWrite`]: tokio::io::AsyncWrite
-//! [`Stream`]: tokio_stream::Stream
+//! [`Stream`]: futures_core::Stream
 //! [`Sink`]: futures_sink::Sink
 //! [`SinkExt::close`]: https://docs.rs/futures/0.3/futures/sink/trait.SinkExt.html#method.close
 //! [`FramedRead`]: struct@crate::codec::FramedRead
@@ -285,3 +285,6 @@ pub use self::length_delimited::{LengthDelimitedCodec, LengthDelimitedCodecError
 
 mod lines_codec;
 pub use self::lines_codec::{LinesCodec, LinesCodecError};
+
+mod any_delimiter_codec;
+pub use self::any_delimiter_codec::{AnyDelimiterCodec, AnyDelimiterCodecError};

tokio-util/src/compat.rs

@@ -215,3 +215,17 @@ where
         tokio::io::AsyncWrite::poll_shutdown(self.project().inner, cx)
     }
 }
+
+#[cfg(unix)]
+impl<T: std::os::unix::io::AsRawFd> std::os::unix::io::AsRawFd for Compat<T> {
+    fn as_raw_fd(&self) -> std::os::unix::io::RawFd {
+        self.inner.as_raw_fd()
+    }
+}
+
+#[cfg(windows)]
+impl<T: std::os::windows::io::AsRawHandle> std::os::windows::io::AsRawHandle for Compat<T> {
+    fn as_raw_handle(&self) -> std::os::windows::io::RawHandle {
+        self.inner.as_raw_handle()
+    }
+}

tokio-util/src/io/reader_stream.rs

@@ -40,7 +40,7 @@ pin_project! {
     ///
     /// [`AsyncRead`]: tokio::io::AsyncRead
     /// [`StreamReader`]: crate::io::StreamReader
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     #[derive(Debug)]
     pub struct ReaderStream<R> {
         // Reader itself.
@@ -58,7 +58,7 @@ impl<R: AsyncRead> ReaderStream<R> {
     /// `Result<Bytes, std::io::Error>`.
     ///
     /// [`AsyncRead`]: tokio::io::AsyncRead
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     pub fn new(reader: R) -> Self {
         ReaderStream {
             reader: Some(reader),

tokio-util/src/io/stream_reader.rs

@@ -51,7 +51,7 @@ pin_project! {
     /// ```
     ///
     /// [`AsyncRead`]: tokio::io::AsyncRead
-    /// [`Stream`]: tokio_stream::Stream
+    /// [`Stream`]: futures_core::Stream
     /// [`ReaderStream`]: crate::io::ReaderStream
     #[derive(Debug)]
     pub struct StreamReader<S, B> {
@@ -93,6 +93,36 @@ where
     }
 }
 
+impl<S, B> StreamReader<S, B> {
+    /// Gets a reference to the underlying stream.
+    ///
+    /// It is inadvisable to directly read from the underlying stream.
+    pub fn get_ref(&self) -> &S {
+        &self.inner
+    }
+
+    /// Gets a mutable reference to the underlying stream.
+    ///
+    /// It is inadvisable to directly read from the underlying stream.
+    pub fn get_mut(&mut self) -> &mut S {
+        &mut self.inner
+    }
+
+    /// Gets a pinned mutable reference to the underlying stream.
+    ///
+    /// It is inadvisable to directly read from the underlying stream.
+    pub fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut S> {
+        self.project().inner
+    }
+
+    /// Consumes this `BufWriter`, returning the underlying stream.
+    ///
+    /// Note that any leftover data in the internal buffer is lost.
+    pub fn into_inner(self) -> S {
+        self.inner
+    }
+}
+
 impl<S, B, E> AsyncRead for StreamReader<S, B>
 where
     S: Stream<Item = Result<B, E>>,

tokio-util/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc(html_root_url = "https://docs.rs/tokio-util/0.6.0")]
 #![allow(clippy::needless_doctest_main)]
 #![warn(
     missing_debug_implementations,
@@ -46,13 +45,14 @@ cfg_rt! {
     pub mod context;
 }
 
+cfg_time! {
+    pub mod time;
+}
+
 pub mod sync;
 
 pub mod either;
 
-#[cfg(feature = "time")]
-pub mod time;
-
 #[cfg(any(feature = "io", feature = "codec"))]
 mod util {
     use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};

tokio-util/src/sync/mod.rs

@@ -4,3 +4,12 @@ mod cancellation_token;
 pub use cancellation_token::{CancellationToken, WaitForCancellationFuture};
 
 mod intrusive_double_linked_list;
+
+mod mpsc;
+pub use mpsc::PollSender;
+
+mod poll_semaphore;
+pub use poll_semaphore::PollSemaphore;
+
+mod reusable_box;
+pub use reusable_box::ReusableBoxFuture;

tokio-util/src/sync/mpsc.rs

@@ -0,0 +1,221 @@
+use futures_core::ready;
+use futures_sink::Sink;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use tokio::sync::mpsc::{error::SendError, Sender};
+
+use super::ReusableBoxFuture;
+
+// This implementation was chosen over something based on permits because to get a
+// `tokio::sync::mpsc::Permit` out of the `inner` future, you must transmute the
+// lifetime on the permit to `'static`.
+
+/// A wrapper around [`mpsc::Sender`] that can be polled.
+///
+/// [`mpsc::Sender`]: tokio::sync::mpsc::Sender
+#[derive(Debug)]
+pub struct PollSender<T> {
+    /// is none if closed
+    sender: Option<Arc<Sender<T>>>,
+    is_sending: bool,
+    inner: ReusableBoxFuture<Result<(), SendError<T>>>,
+}
+
+// By reusing the same async fn for both Some and None, we make sure every
+// future passed to ReusableBoxFuture has the same underlying type, and hence
+// the same size and alignment.
+async fn make_future<T>(data: Option<(Arc<Sender<T>>, T)>) -> Result<(), SendError<T>> {
+    match data {
+        Some((sender, value)) => sender.send(value).await,
+        None => unreachable!(
+            "This future should not be pollable, as is_sending should be set to false."
+        ),
+    }
+}
+
+impl<T: Send + 'static> PollSender<T> {
+    /// Create a new `PollSender`.
+    pub fn new(sender: Sender<T>) -> Self {
+        Self {
+            sender: Some(Arc::new(sender)),
+            is_sending: false,
+            inner: ReusableBoxFuture::new(make_future(None)),
+        }
+    }
+
+    /// Start sending a new item.
+    ///
+    /// This method panics if a send is currently in progress. To ensure that no
+    /// send is in progress, call `poll_send_done` first until it returns
+    /// `Poll::Ready`.
+    ///
+    /// If this method returns an error, that indicates that the channel is
+    /// closed. Note that this method is not guaranteed to return an error if
+    /// the channel is closed, but in that case the error would be reported by
+    /// the first call to `poll_send_done`.
+    pub fn start_send(&mut self, value: T) -> Result<(), SendError<T>> {
+        if self.is_sending {
+            panic!("start_send called while not ready.");
+        }
+        match self.sender.clone() {
+            Some(sender) => {
+                self.inner.set(make_future(Some((sender, value))));
+                self.is_sending = true;
+                Ok(())
+            }
+            None => Err(SendError(value)),
+        }
+    }
+
+    /// If a send is in progress, poll for its completion. If no send is in progress,
+    /// this method returns `Poll::Ready(Ok(()))`.
+    ///
+    /// This method can return the following values:
+    ///
+    ///  - `Poll::Ready(Ok(()))` if the in-progress send has been completed, or there is
+    ///    no send in progress (even if the channel is closed).
+    ///  - `Poll::Ready(Err(err))` if the in-progress send failed because the channel has
+    ///    been closed.
+    ///  - `Poll::Pending` if a send is in progress, but it could not complete now.
+    ///
+    /// When this method returns `Poll::Pending`, the current task is scheduled
+    /// to receive a wakeup when the message is sent, or when the entire channel
+    /// is closed (but not if just this sender is closed by
+    /// `close_this_sender`). Note that on multiple calls to `poll_send_done`,
+    /// only the `Waker` from the `Context` passed to the most recent call is
+    /// scheduled to receive a wakeup.
+    ///
+    /// If this method returns `Poll::Ready`, then `start_send` is guaranteed to
+    /// not panic.
+    pub fn poll_send_done(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), SendError<T>>> {
+        if !self.is_sending {
+            return Poll::Ready(Ok(()));
+        }
+
+        let result = self.inner.poll(cx);
+        if result.is_ready() {
+            self.is_sending = false;
+        }
+        if let Poll::Ready(Err(_)) = &result {
+            self.sender = None;
+        }
+        result
+    }
+
+    /// Check whether the channel is ready to send more messages now.
+    ///
+    /// If this method returns `true`, then `start_send` is guaranteed to not
+    /// panic.
+    ///
+    /// If the channel is closed, this method returns `true`.
+    pub fn is_ready(&self) -> bool {
+        !self.is_sending
+    }
+
+    /// Check whether the channel has been closed.
+    pub fn is_closed(&self) -> bool {
+        match &self.sender {
+            Some(sender) => sender.is_closed(),
+            None => true,
+        }
+    }
+
+    /// Clone the underlying `Sender`.
+    ///
+    /// If this method returns `None`, then the channel is closed. (But it is
+    /// not guaranteed to return `None` if the channel is closed.)
+    pub fn clone_inner(&self) -> Option<Sender<T>> {
+        self.sender.as_ref().map(|sender| (&**sender).clone())
+    }
+
+    /// Access the underlying `Sender`.
+    ///
+    /// If this method returns `None`, then the channel is closed. (But it is
+    /// not guaranteed to return `None` if the channel is closed.)
+    pub fn inner_ref(&self) -> Option<&Sender<T>> {
+        self.sender.as_deref()
+    }
+
+    // This operation is supported because it is required by the Sink trait.
+    /// Close this sender. No more messages can be sent from this sender.
+    ///
+    /// Note that this only closes the channel from the view-point of this
+    /// sender. The channel remains open until all senders have gone away, or
+    /// until the [`Receiver`] closes the channel.
+    ///
+    /// If there is a send in progress when this method is called, that send is
+    /// unaffected by this operation, and `poll_send_done` can still be called
+    /// to complete that send.
+    ///
+    /// [`Receiver`]: tokio::sync::mpsc::Receiver
+    pub fn close_this_sender(&mut self) {
+        self.sender = None;
+    }
+
+    /// Abort the current in-progress send, if any.
+    ///
+    /// Returns `true` if a send was aborted.
+    pub fn abort_send(&mut self) -> bool {
+        if self.is_sending {
+            self.inner.set(make_future(None));
+            self.is_sending = false;
+            true
+        } else {
+            false
+        }
+    }
+}
+
+impl<T> Clone for PollSender<T> {
+    /// Clones this `PollSender`. The resulting clone will not have any
+    /// in-progress send operations, even if the current `PollSender` does.
+    fn clone(&self) -> PollSender<T> {
+        Self {
+            sender: self.sender.clone(),
+            is_sending: false,
+            inner: ReusableBoxFuture::new(async { unreachable!() }),
+        }
+    }
+}
+
+impl<T: Send + 'static> Sink<T> for PollSender<T> {
+    type Error = SendError<T>;
+
+    /// This is equivalent to calling [`poll_send_done`].
+    ///
+    /// [`poll_send_done`]: PollSender::poll_send_done
+    fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Pin::into_inner(self).poll_send_done(cx)
+    }
+
+    /// This is equivalent to calling [`poll_send_done`].
+    ///
+    /// [`poll_send_done`]: PollSender::poll_send_done
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Pin::into_inner(self).poll_send_done(cx)
+    }
+
+    /// This is equivalent to calling [`start_send`].
+    ///
+    /// [`start_send`]: PollSender::start_send
+    fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> {
+        Pin::into_inner(self).start_send(item)
+    }
+
+    /// This method will first flush the `PollSender`, and then close it by
+    /// calling [`close_this_sender`].
+    ///
+    /// If a send fails while flushing because the [`Receiver`] has gone away,
+    /// then this function returns an error. The channel is still successfully
+    /// closed in this situation.
+    ///
+    /// [`close_this_sender`]: PollSender::close_this_sender
+    /// [`Receiver`]: tokio::sync::mpsc::Receiver
+    fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        ready!(self.as_mut().poll_flush(cx))?;
+
+        Pin::into_inner(self).close_this_sender();
+        Poll::Ready(Ok(()))
+    }
+}