Start work on compat feature

This makes it consistent with `TcpListener::poll_accept` and other
public poll methods the library provides. This particular method is
useful for writing generic code that accepts connections since async
functions can't easily be used with traits. It is possible to
generically accept connections with `Incoming`, however, this doesn't
return the incoming `SocketAddr`.

.cirrus.yml

@@ -1,5 +1,5 @@
 freebsd_instance:
-  image: freebsd-12-0-release-amd64
+  image: freebsd-12-1-release-amd64
 
 # Test FreeBSD in a full VM on cirrus-ci.com.  Test the i686 target too, in the
 # same VM.  The binary will be built in 32-bit mode, but will execute on a
@@ -19,18 +19,6 @@ task:
     - |
       echo "~~~~ rustc --version ~~~~"
       rustc --version
-
-      # Remove any existing patch statements
-      mv Cargo.toml Cargo.toml.bck
-      sed -n '/\[patch.crates-io\]/q;p' Cargo.toml.bck > Cargo.toml
-
-      # Patch all crates
-      cat ci/patch.toml >> Cargo.toml
-
-      # Print `Cargo.toml` for debugging
-      echo "~~~~ Cargo.toml ~~~~"
-      cat Cargo.toml
-      echo "~~~~~~~~~~~~~~~~~~~~"
   test_script:
     - . $HOME/.cargo/env
     - cargo test --all
@@ -39,4 +27,4 @@ task:
   # i686_test_script:
   #   - . $HOME/.cargo/env
   #   - |
-  #     cargo test --all --exclude tokio-tls --exclude tokio-macros --target i686-unknown-freebsd
+  #     cargo test --all --exclude tokio-macros --target i686-unknown-freebsd

.github/ISSUE_TEMPLATE.md

@@ -1,51 +0,0 @@
-<!--
-Thank you for reporting an issue.
-
-Please fill in as much of the template below as you're able.
--->
-
-## Version
-
-<!--
-List the versions of all `tokio` crates you are using. The easiest way to get
-this information is using `cargo-tree`.
-
-`cargo install cargo-tree`
-(see install here: https://github.com/sfackler/cargo-tree)
-
-Then:
-
-`cargo tree | grep tokio`
--->
-
-## Platform
-
-<!---
-Output of `uname -a` (UNIX), or version and 32 or 64-bit (Windows)
--->
-
-## Subcrates
-
-<!--
-If known, please specify the affected Tokio sub crates. Otherwise, delete this
-section.
--->
-
-## Description
-
-<!--
-
-Enter your issue details below this comment.
-
-One way to structure the description:
-
-<short summary of the bug>
-
-I tried this code:
-
-<code sample that causes the bug>
-
-I expected to see this happen: <explanation>
-
-Instead, this happened: <explanation>
--->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: A-tokio, C-bug
+assignees: ''
+
+---
+
+**Version**
+List the versions of all `tokio` crates you are using. The easiest way to get
+this information is using `cargo-tree`.
+
+`cargo install cargo-tree`
+(see install here: https://github.com/sfackler/cargo-tree)
+
+Then:
+
+`cargo tree | grep tokio`
+
+**Platform**
+The output of `uname -a` (UNIX), or version and 32 or 64-bit (Windows)
+
+**Description**
+Enter your issue details here.
+One way to structure the description:
+
+[short summary of the bug]
+
+I tried this code:
+
+[code sample that causes the bug]
+
+I expected to see this happen: [explanation]
+
+Instead, this happened: [explanation]

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: A-tokio, C-feature-request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,16 @@
+---
+name: Question
+about: Please use the discussions tab for questions
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please post your question as a discussion here:
+https://github.com/tokio-rs/tokio/discussions
+
+
+You may also be able to find help here:
+https://discord.gg/tokio
+https://users.rust-lang.org/

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,6 +5,9 @@ the requirements below.
 Bug fixes and new features should include tests.
 
 Contributors guide: https://github.com/tokio-rs/tokio/blob/master/CONTRIBUTING.md
+
+The contributors guide includes instructions for running rustfmt and building the
+documentation, which requires special commands beyond `cargo fmt` and `cargo doc`.
 -->
 
 ## Motivation

.github/workflows/audit.yml

@@ -0,0 +1,22 @@
+name: Security Audit
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - '**/Cargo.toml'
+  schedule:
+    - cron: '0 2 * * *' # run at 2 AM UTC
+
+jobs:
+  security-audit:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'ci skip')"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Audit Check
+        uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -0,0 +1,244 @@
+on:
+  push:
+    branches: ["v0.2.x"]
+  pull_request:
+    branches: ["v0.2.x"]
+
+name: CI
+
+env:
+  RUSTFLAGS: -Dwarnings
+  RUST_BACKTRACE: 1
+  nightly: nightly-2020-09-21
+  minrust: 1.39.0
+
+jobs:
+  # Depends on all action sthat are required for a "successful" CI run.
+  tests-pass:
+    name: all systems go
+    runs-on: ubuntu-latest
+    needs:
+      - test
+      - test-unstable
+      - miri
+      - cross
+      - features
+      - minrust
+      - fmt
+      - clippy
+      - docs
+      - loom
+    steps:
+      - run: exit 0
+
+  test:
+    name: test tokio full
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup update stable
+      - name: Install cargo-hack
+        run: cargo install cargo-hack
+
+      # Run `tokio` with `full` features. This excludes testing utilities which
+      # can alter the runtime behavior of Tokio.
+      - name: test tokio full
+        run: cargo test --features full
+        working-directory: tokio
+
+      # Check `tokio` with `full + parking_lot` to make sure it compiles.
+      - name: check tokio full,parking_lot
+        run: cargo check --features full,parking_lot
+        working-directory: tokio
+
+      # Test **all** crates in the workspace with all features.
+      - name: test all --all-features
+        run: cargo test --workspace --all-features
+
+      # Run integration tests for each feature
+      - name: test tests-integration --each-feature
+        run: cargo hack test --each-feature
+        working-directory: tests-integration
+
+      # Run macro build tests
+      - name: test tests-build --each-feature
+        run: cargo hack test --each-feature
+        working-directory: tests-build
+
+  test-unstable:
+    name: test tokio full --unstable
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+          - ubuntu-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup update stable
+
+      # Run `tokio` with "unstable" cfg flag.
+      - name: test tokio full --cfg unstable
+        run: cargo test --features full
+        working-directory: tokio
+        env:
+          RUSTFLAGS: '--cfg tokio_unstable'
+
+  miri:
+    name: miri
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.nightly }}
+          override: true
+      - name: Install Miri
+        run: |
+          set -e
+          rustup component add miri
+          cargo miri setup
+          rm -rf tokio/tests
+
+      - name: miri
+        run: cargo miri test --features rt-core,rt-threaded,rt-util,sync task
+        working-directory: tokio
+
+  cross:
+    name: cross
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - i686-unknown-linux-gnu
+          - powerpc-unknown-linux-gnu
+          - powerpc64-unknown-linux-gnu
+          - mips-unknown-linux-gnu
+          - arm-linux-androideabi
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: ${{ matrix.target }}
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: check
+          args: --workspace --target ${{ matrix.target }}
+
+  features:
+    name: features
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.nightly }}
+          override: true
+      - name: Install cargo-hack
+        run: cargo install cargo-hack
+
+      - name: check --each-feature
+        run: cargo hack check --all --each-feature -Z avoid-dev-deps
+
+      # Try with unstable feature flags
+      - name: check --each-feature --unstable
+        run: cargo hack check --all --each-feature -Z avoid-dev-deps
+        env:
+          RUSTFLAGS: --cfg tokio_unstable
+
+  minrust:
+    name: minrust
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.minrust }}
+          override: true
+
+      - name: "test --workspace --all-features"
+        run: cargo check --workspace --all-features
+
+  fmt:
+    name: fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup update stable
+      - name: Install rustfmt
+        run: rustup component add rustfmt
+
+      # Check fmt
+      - name: "rustfmt --check"
+        # Workaround for rust-lang/cargo#7732
+        run: |
+          if ! rustfmt --check --edition 2018 $(find . -name '*.rs' -print); then
+            printf "Please run \`rustfmt --edition 2018 \$(find . -name '*.rs' -print)\` to fix rustfmt errors.\nSee CONTRIBUTING.md for more details.\n" >&2
+            exit 1
+          fi
+
+  clippy:
+    name: clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup update stable
+      - name: Install clippy
+        run: rustup component add clippy
+
+      # Run clippy
+      - name: "clippy --all"
+        run: cargo clippy --all --tests
+
+  docs:
+    name: docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.nightly }}
+          override: true
+
+      - name: "doc --lib --all-features"
+        run: cargo doc --lib --no-deps --all-features
+        env:
+          RUSTDOCFLAGS: --cfg docsrs
+
+  loom:
+    name: loom
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        scope:
+          - --skip loom_pool
+          - loom_pool::group_a
+          - loom_pool::group_b
+          - loom_pool::group_c
+          - loom_pool::group_d
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Rust
+        run: rustup update stable
+
+      - name: loom ${{ matrix.scope }}
+        run: cargo test --lib --release --features full -- --nocapture $SCOPE
+        working-directory: tokio
+        env:
+          RUSTFLAGS: --cfg loom --cfg tokio_unstable
+          LOOM_MAX_PREEMPTIONS: 2
+          SCOPE: ${{ matrix.scope }}

.github/workflows/pr-audit.yml

@@ -0,0 +1,32 @@
+name: Pull Request Security Audit
+
+on:
+  push:
+    paths:
+      - '**/Cargo.toml'
+  pull_request:
+    paths:
+      - '**/Cargo.toml'
+
+jobs:
+  security-audit:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'ci skip')"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install cargo-audit
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: cargo-audit
+
+      - name: Generate lockfile
+        uses: actions-rs/cargo@v1
+        with:
+          command: generate-lockfile
+
+      - name: Audit dependencies
+        uses: actions-rs/cargo@v1
+        with:
+          command: audit

CODE_OF_CONDUCT.md

@@ -0,0 +1,7 @@
+# Code of Conduct
+
+The Tokio project adheres to the [Rust Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct). This describes the minimum behavior expected from all contributors.
+
+## Enforcement
+
+Instances of violations of the Code of Conduct can be reported by contacting the project team at [[email protected]](mailto:[email protected]).

CONTRIBUTING.md

@@ -15,12 +15,14 @@ It should be considered a map to help you navigate the process.
 The [dev channel][dev] is available for any concerns not covered in this guide, please join
 us!
 
-[dev]: https://discord.gg/6yGkFeN
+[dev]: https://discord.gg/tokio
 
 ## Conduct
 
 The Tokio project adheres to the [Rust Code of Conduct][coc]. This describes
-the _minimum_ behavior expected from all contributors.
+the _minimum_ behavior expected from all contributors. Instances of violations of the
+Code of Conduct can be reported by contacting the project team at
+[[email protected]](mailto:[email protected]).
 
 [coc]: https://github.com/rust-lang/rust/blob/master/CODE_OF_CONDUCT.md
 
@@ -29,8 +31,8 @@ the _minimum_ behavior expected from all contributors.
 For any issue, there are fundamentally three ways an individual can contribute:
 
 1. By opening the issue for discussion: For instance, if you believe that you
-   have uncovered a bug in Tokio, creating a new issue in the tokio-rs/tokio
-   issue tracker is the way to report it.
+   have discovered a bug in Tokio, creating a new issue in [the tokio-rs/tokio
+   issue tracker][issue] is the way to report it.
 
 2. By helping to triage the issue: This can be done by providing
    supporting details (a test case that demonstrates a bug), providing
@@ -42,21 +44,25 @@ For any issue, there are fundamentally three ways an individual can contribute:
    often, by opening a Pull Request that changes some bit of something in
    Tokio in a concrete and reviewable manner.
 
+[issue]: https://github.com/tokio-rs/tokio/issues
+
 **Anybody can participate in any stage of contribution**. We urge you to
 participate in the discussion around bugs and participate in reviewing PRs.
 
 ### Asking for General Help
 
 If you have reviewed existing documentation and still have questions or are
-having problems, you can open an issue asking for help.
+having problems, you can [open a discussion] asking for help.
 
 In exchange for receiving help, we ask that you contribute back a documentation
 PR that helps others avoid the problems that you encountered.
 
+[open a discussion]: https://github.com/tokio-rs/tokio/discussions/new
+
 ### Submitting a Bug Report
 
-When opening a new issue in the Tokio issue tracker, users will be presented
-with a [basic template][template] that should be filled in. If you believe that you have
+When opening a new issue in the Tokio issue tracker, you will be presented
+with a basic template that should be filled in. If you believe that you have
 uncovered a bug, please fill out this form, following the template to the best
 of your ability. Do not worry if you cannot answer every detail, just fill in
 what you can.
@@ -72,7 +78,6 @@ cases should be limited, as much as possible, to using only Tokio APIs.
 See [How to create a Minimal, Complete, and Verifiable example][mcve].
 
 [mcve]: https://stackoverflow.com/help/mcve
-[template]: .github/PULL_REQUEST_TEMPLATE.md
 
 ### Triaging a Bug Report
 
@@ -112,6 +117,44 @@ usually a good idea to first open an issue describing the change to solicit
 feedback and guidance. This will increase the likelihood of the PR getting
 merged.
 
+### Cargo Commands
+
+Due to the extensive use of features in Tokio, you will often need to add extra
+arguments to many common cargo commands. This section lists some commonly needed
+commands.
+
+Some commands just need the `--all-features` argument:
+```
+cargo build --all-features
+cargo check --all-features
+cargo test --all-features
+```
+When building documentation normally, the markers that list the features
+required for various parts of Tokio are missing. To build the documentation
+correctly, use this command:
+```
+RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features
+```
+The `cargo fmt` command does not work on the Tokio codebase. You can use the
+command below instead:
+
+```
+# Mac or Linux
+rustfmt --check --edition 2018 $(find . -name '*.rs' -print)
+
+# Powershell
+Get-ChildItem . -Filter "*.rs" -Recurse | foreach { rustfmt --check --edition 2018 $_.FullName }
+```
+The `--check` argument prints the things that need to be fixed. If you remove
+it, `rustfmt` will update your files locally instead.
+
+You can run loom tests with
+```
+cd tokio # tokio crate in workspace
+LOOM_MAX_PREEMPTIONS=1 RUSTFLAGS="--cfg loom" \
+    cargo test --lib --release --features full -- --test-threads=1 --nocapture
+```
+
 ### Tests
 
 If the change being proposed alters code (as opposed to only documentation for
@@ -217,7 +260,7 @@ That said, if you have a number of commits that are "checkpoints" and don't
 represent a single logical change, please squash those together.
 
 Note that multiple commits often get squashed when they are landed (see the
-notes about [commit squashing]).
+notes about [commit squashing](#commit-squashing)).
 
 #### Commit message guidelines
 
@@ -288,7 +331,7 @@ in order to evaluate whether the changes are correct and necessary.
 Keep an eye out for comments from code owners to provide guidance on conflicting
 feedback.
 
-**Once the PR is open, do not rebase the commits**. See [Commit Squashing] for
+**Once the PR is open, do not rebase the commits**. See [Commit Squashing](#commit-squashing) for
 more details.
 
 ### Commit Squashing
@@ -382,6 +425,83 @@ _Adapted from the [Node.js contributing guide][node]_.
 [hiding-a-comment]: https://help.github.com/articles/managing-disruptive-comments/#hiding-a-comment
 [documentation test]: https://doc.rust-lang.org/rustdoc/documentation-tests.html
 
+## Keeping track of issues and PRs
+
+The Tokio GitHub repository has a lot of issues and PRs, which is not easy to
+keep track of. This section explains the meaning of various labels, as well as
+our [GitHub project][project]. The section is primarily targeted at maintainers.
+
+**Area.** The area label describes the crates relevant to this issue or PR.
+
+ - **A-tokio** This issue concerns the main Tokio crate.
+ - **A-tokio-util** This issue concerns the `tokio-util` crate.
+ - **A-tokio-tls** This issue concerns the `tokio-tls` crate. Only used for
+   older issues, as the crate has been moved to another repository.
+ - **A-tokio-test** The issue concerns the `tokio-test` crate.
+ - **A-tokio-macros** This issue concerns the `tokio-macros` crate. Should only
+   be used for the procedural macros, and not `join!` or `select!`.
+ - **A-ci** This issue concerns our GitHub Actions setup.
+
+**Category.** The category label describes the category.
+
+ - **C-bug** This is a bug-report. Bug-fix PRs use `C-enhancement` instead.
+ - **C-enhancement** This is a PR that adds a new features.
+ - **C-maintenance** This is an issue or PR about stuff such as documentation,
+   GitHub Actions or code quality.
+ - **C-feature-request** This is a feature request. Implementations of feature
+   requests use `C-enhancement` instead.
+ - **C-feature-accepted** If you submit a PR for this feature request, we wont
+   close it with the reason "we don't want this". Issues with this label should
+   also have the `C-feature-request` label.
+ - **C-musing** Stuff like tracking issues or roadmaps. "musings about a better
+   world"
+ - **C-proposal** A proposal of some kind, and a request for comments.
+ - **C-question** A user question. Large overlap with GitHub discussions.
+ - **C-request** A non-feature request, e.g. "please add deprecation notices to
+   `-alpha.*` versions of crates"
+
+**Call for participation.** I don't know why it's called `E-`. Many issues are
+missing a difficulty rating, and you should feel free to add one.
+
+ - **E-help-wanted** Stuff where we want help. Often seen together with `C-bug`
+   or `C-feature-accepted`.
+ - **E-easy** This is easy, ranging from quick documentation fixes to stuff you
+   can do after reading the tutorial on our website.
+ - **E-medium** This is not `E-easy` or `E-hard`.
+ - **E-hard** This either involves very tricky code, is something we don't know
+   how to solve, or is difficult for some other reason.
+ - **E-needs-mvce** This bug is missing a minimal complete and verifiable
+   example.
+
+**Module.** A more fine groaned categorization than area.
+
+ - **M-blocking** Things relevant to `spawn_blocking`, `block_in_place`.
+ - **M-codec** The `tokio_util::codec` module.
+ - **M-compat** The `tokio_util::compat` module.
+ - **M-coop** Things relevant to coop.
+ - **M-fs** The `tokio::fs` module.
+ - **M-io** The `tokio::io` module.
+ - **M-macros** Issues about any kind of macro.
+ - **M-net** The `tokio::net` module.
+ - **M-process** The `tokio::process` module.
+ - **M-runtime** The `tokio::runtime` module.
+ - **M-signal** The `tokio::signal` module.
+ - **M-stream** The `tokio::stream` module.
+ - **M-sync** The `tokio::sync` module.
+ - **M-task** The `tokio::task` module.
+ - **M-time** The `tokio::time` module.
+ - **M-tracing** Tracing support in Tokio.
+
+**Topic.** Some extra information.
+
+ - **T-docs** This is about documentation.
+ - **T-performance** This is about performance.
+ - **T-v0.1.x** This is about old Tokio.
+
+Any label not listed here is not in active use.
+
+[project]: https://github.com/orgs/tokio-rs/projects/1
+
 ## Releasing
 
 Since the Tokio project consists of a number of crates, many of which depend on

Cargo.toml

@@ -4,10 +4,10 @@ members = [
   "tokio",
   "tokio-macros",
   "tokio-test",
-  "tokio-tls",
   "tokio-util",
 
   # Internal
+  "benches",
   "examples",
   "tests-build",
   "tests-integration",

README.md

@@ -20,17 +20,17 @@ the Rust programming language. It is:
 [crates-badge]: https://img.shields.io/crates/v/tokio.svg
 [crates-url]: https://crates.io/crates/tokio
 [mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg
-[mit-url]: LICENSE
+[mit-url]: https://github.com/tokio-rs/tokio/blob/master/LICENSE
 [azure-badge]: https://dev.azure.com/tokio-rs/Tokio/_apis/build/status/tokio-rs.tokio?branchName=master
 [azure-url]: https://dev.azure.com/tokio-rs/Tokio/_build/latest?definitionId=1&branchName=master
 [discord-badge]: https://img.shields.io/discord/500028886025895936.svg?logo=discord&style=flat-square
-[discord-url]: https://discord.gg/6yGkFeN
+[discord-url]: https://discord.gg/tokio
 
 [Website](https://tokio.rs) |
-[Guides](https://tokio.rs/docs/) |
+[Guides](https://tokio.rs/tokio/tutorial) |
 [API Docs](https://docs.rs/tokio/latest/tokio) |
 [Roadmap](https://github.com/tokio-rs/tokio/blob/master/ROADMAP.md) |
-[Chat](https://discord.gg/6yGkFeN)
+[Chat](https://discord.gg/tokio)
 
 ## Overview
 
@@ -90,23 +90,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 ```
 
-More examples can be found [here](examples). Note that the `master` branch
-is currently being updated to use `async` / `await`. The examples are
-not fully ported. Examples for stable Tokio can be found
-[here](https://github.com/tokio-rs/tokio/tree/v0.1.x/tokio/examples).
+More examples can be found [here][examples]. For a larger "real world" example, see the
+[mini-redis] repository.
 
+[examples]: https://github.com/tokio-rs/tokio/tree/master/examples
+[mini-redis]: https://github.com/tokio-rs/mini-redis/
+
+To see a list of the available features flags that can be enabled, check our
+[docs][feature-flag-docs].
 
 ## Getting Help
 
 First, see if the answer to your question can be found in the [Guides] or the
 [API documentation]. If the answer is not there, there is an active community in
 the [Tokio Discord server][chat]. We would be happy to try to answer your
-question. Last, if that doesn't work, try opening an [issue] with the question.
+question. You can also ask your question on [the discussions page][discussions].
 
-[Guides]: https://tokio.rs/docs/
+[Guides]: https://tokio.rs/tokio/tutorial
 [API documentation]: https://docs.rs/tokio/latest/tokio
-[chat]: https://discord.gg/6yGkFeN
-[issue]: https://github.com/tokio-rs/tokio/issues/new
+[chat]: https://discord.gg/tokio
+[discussions]: https://github.com/tokio-rs/tokio/discussions
+[feature-flag-docs]: https://docs.rs/tokio/#feature-flags
 
 ## Contributing
 
@@ -114,36 +118,54 @@ question. Last, if that doesn't work, try opening an [issue] with the question.
 you! We have a [contributing guide][guide] to help you get involved in the Tokio
 project.
 
-[guide]: CONTRIBUTING.md
+[guide]: https://github.com/tokio-rs/tokio/blob/master/CONTRIBUTING.md
 
 ## Related Projects
 
 In addition to the crates in this repository, the Tokio project also maintains
 several other libraries, including:
 
+* [`hyper`]: A fast and correct HTTP/1.1 and HTTP/2 implementation for Rust.
+
+* [`tonic`]: A gRPC over HTTP/2 implementation focused on high performance, interoperability, and flexibility.
+
+* [`warp`]: A super-easy, composable, web server framework for warp speeds.
+
+* [`tower`]: A library of modular and reusable components for building robust networking clients and servers.
+
 * [`tracing`] (formerly `tokio-trace`): A framework for application-level
   tracing and async-aware diagnostics.
 
+* [`rdbc`]: A Rust database connectivity library for MySQL, Postgres and SQLite.
+
 * [`mio`]: A low-level, cross-platform abstraction over OS I/O APIs that powers
   `tokio`.
 
 * [`bytes`]: Utilities for working with bytes, including efficient byte buffers.
 
+* [`loom`]: A testing tool for concurrent Rust code
+
+[`warp`]: https://github.com/seanmonstar/warp
+[`hyper`]: https://github.com/hyperium/hyper
+[`tonic`]: https://github.com/hyperium/tonic
+[`tower`]: https://github.com/tower-rs/tower
+[`loom`]: https://github.com/tokio-rs/loom
+[`rdbc`]: https://github.com/tokio-rs/rdbc
 [`tracing`]: https://github.com/tokio-rs/tracing
 [`mio`]: https://github.com/tokio-rs/mio
 [`bytes`]: https://github.com/tokio-rs/bytes
 
 ## Supported Rust Versions
 
-Tokio is built against the latest stable, nightly, and beta Rust releases. The
-minimum version supported is the stable release from three months before the
-current stable release version. For example, if the latest stable Rust is 1.29,
-the minimum version supported is 1.26. The current Tokio version is not
-guaranteed to build on Rust versions earlier than the minimum supported version.
+Tokio is built against the latest stable release. The minimum supported version is 1.39.
+The current Tokio version is not guaranteed to build on Rust versions earlier than the
+minimum supported version.
 
 ## License
 
-This project is licensed under the [MIT license](LICENSE).
+This project is licensed under the [MIT license].
+
+[MIT license]: https://github.com/tokio-rs/tokio/blob/master/LICENSE
 
 ### Contribution
 

ROADMAP.md

@@ -26,13 +26,13 @@ The Tokio 1.0 release will be **no later** than Q3 2020. It will also come with
 
 When Tokio 1.0 is released in Q3 2020, on-going support, security fixes, and
 critical bug fixes are guaranteed until **at least** Q3 2025. Tokio 2.0 will not
-be released until **at least** Q3 2023 (though, ideally there will never been a
+be released until **at least** Q3 2023 (though, ideally there will never be a
 Tokio 2.0 release).
 
 ### How to get there
 
 While Tokio 0.1 probably should have been a 1.0, Tokio 0.2 will be a **true**
-0.2 release. There will breaking change releases every 2 ~ 3 months until 1.0.
+0.2 release. There will be breaking change releases every 2 ~ 3 months until 1.0.
 These changes will be **much** smaller than going from 0.1 -> 0.2. It is
 expected that the 1.0 release will look a lot like 0.2.
 

SECURITY.md

@@ -0,0 +1,13 @@
+## Report a security issue
+
+The Tokio project team welcomes security reports and is committed to providing prompt attention to security issues. Security issues should be reported privately via [[email protected]](mailto:[email protected]). Security issues should not be reported via the public Github Issue tracker.
+
+## Vulnerability coordination
+
+Remediation of security vulnerabilities is prioritized by the project team. The project team coordinates remediation with third-party project stakeholders via [Github Security Advisories](https://help.github.com/en/github/managing-security-vulnerabilities/about-github-security-advisories). Third-party stakeholders may include the reporter of the issue, affected direct or indirect users of Tokio, and maintainers of upstream dependencies if applicable.
+
+Downstream project maintainers and Tokio users can request participation in coordination of applicable security issues by sending your contact email address, Github username(s) and any other salient information to [[email protected]](mailto:[email protected]). Participation in security issue coordination processes is at the discretion of the Tokio team.
+
+## Security advisories
+
+The project team is committed to transparency in the security issue disclosure process. The Tokio team announces security issues via [project Github Release notes](https://github.com/tokio-rs/tokio/releases) and the [RustSec advisory database](https://github.com/RustSec/advisory-db) (i.e. `cargo-audit`).

azure-pipelines.yml

@@ -1,108 +0,0 @@
-trigger: ["master"]
-pr: ["master"]
-
-variables:
-  RUSTFLAGS: -Dwarnings
-  nightly: nightly-2019-11-16
-
-jobs:
-# Test top level crate
-- template: ci/azure-test-stable.yml
-  parameters:
-    name: test_tokio
-    rust: stable
-    displayName: Test tokio
-    cross: true
-    crates:
-      - tokio
-      - tests-integration
-
-# Test sub crates
-- template: ci/azure-test-stable.yml
-  parameters:
-    name: test_linux
-    displayName: Test sub crates -
-    rust: stable
-    crates:
-      - tokio-macros
-      - tokio-test
-      - tokio-tls
-      - tokio-util
-      - examples
-
-# Run tests from `tests-build`. This requires a different process
-- template: ci/azure-test-build.yml
-  parameters:
-    name: test_build
-    displayName: Test build permutations
-    rust: stable
-
-# Run loom tests
-- template: ci/azure-loom.yml
-  parameters:
-    name: loom
-    rust: stable
-    crates:
-      - tokio
-
-# Try cross compiling
-- template: ci/azure-cross-compile.yml
-  parameters:
-    name: cross
-    rust: stable
-
-# Check each feature works properly
-- template: ci/azure-check-features.yml
-  parameters:
-    rust: $(nightly)
-    name: check_features
-
-# This represents the minimum Rust version supported by
-# Tokio. Updating this should be done in a dedicated PR and
-# cannot be greater than two 0.x releases prior to the
-# current stable.
-#
-# Tests are not run as tests may require newer versions of
-# rust.
-- template: ci/azure-check-minrust.yml
-  parameters:
-    name: minrust
-    rust: 1.39.0
-
-# Check formatting
-- template: ci/azure-rustfmt.yml
-  parameters:
-    rust: stable
-    name: rustfmt
-
-# Apply clippy lints to all crates
-- template: ci/azure-clippy.yml
-  parameters:
-    rust: stable
-    name: clippy
-
-# Check doc generation
-- template: ci/azure-check-docs.yml
-  parameters:
-    rust: $(nightly)
-    name: docs
-
-# - template: ci/azure-tsan.yml
-#   parameters:
-#     name: tsan
-#     rust: stable
-
-- template: ci/azure-deploy-docs.yml
-  parameters:
-    rust: stable
-    dependsOn:
-      - rustfmt
-      - clippy
-      - test_tokio
-      - test_linux
-      - test_build
-      - loom
-      - cross
-      - minrust
-      - check_features
-#      - tsan

benches/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "benches"
+version = "0.0.0"
+publish = false
+edition = "2018"
+
+[dependencies]
+tokio = { version = "0.2.0", path = "../tokio", features = ["full"] }
+bencher = "0.1.5"
+
+[[bench]]
+name = "spawn"
+path = "spawn.rs"
+harness = false
+
+[[bench]]
+name = "mpsc"
+path = "mpsc.rs"
+harness = false
+
+[[bench]]
+name = "scheduler"
+path = "scheduler.rs"
+harness = false
+
+
+[[bench]]
+name = "sync_rwlock"
+path = "sync_rwlock.rs"
+harness = false
+
+[[bench]]
+name = "sync_semaphore"
+path = "sync_semaphore.rs"
+harness = false

benches/mpsc.rs

@@ -0,0 +1,188 @@
+use bencher::{black_box, Bencher};
+use tokio::sync::mpsc;
+
+type Medium = [usize; 64];
+type Large = [Medium; 64];
+
+fn create_1_medium(b: &mut Bencher) {
+    b.iter(|| {
+        black_box(&mpsc::channel::<Medium>(1));
+    });
+}
+
+fn create_100_medium(b: &mut Bencher) {
+    b.iter(|| {
+        black_box(&mpsc::channel::<Medium>(100));
+    });
+}
+
+fn create_100_000_medium(b: &mut Bencher) {
+    b.iter(|| {
+        black_box(&mpsc::channel::<Medium>(100_000));
+    });
+}
+
+fn send_medium(b: &mut Bencher) {
+    b.iter(|| {
+        let (mut tx, mut rx) = mpsc::channel::<Medium>(1000);
+
+        let _ = tx.try_send([0; 64]);
+
+        rx.try_recv().unwrap();
+    });
+}
+
+fn send_large(b: &mut Bencher) {
+    b.iter(|| {
+        let (mut tx, mut rx) = mpsc::channel::<Large>(1000);
+
+        let _ = tx.try_send([[0; 64]; 64]);
+
+        rx.try_recv().unwrap();
+    });
+}
+
+fn contention_bounded(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    b.iter(|| {
+        rt.block_on(async move {
+            let (tx, mut rx) = mpsc::channel::<usize>(1_000_000);
+
+            for _ in 0..5 {
+                let mut tx = tx.clone();
+                tokio::spawn(async move {
+                    for i in 0..1000 {
+                        tx.send(i).await.unwrap();
+                    }
+                });
+            }
+
+            for _ in 0..1_000 * 5 {
+                let _ = rx.recv().await;
+            }
+        })
+    });
+}
+
+fn contention_bounded_full(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    b.iter(|| {
+        rt.block_on(async move {
+            let (tx, mut rx) = mpsc::channel::<usize>(100);
+
+            for _ in 0..5 {
+                let mut tx = tx.clone();
+                tokio::spawn(async move {
+                    for i in 0..1000 {
+                        tx.send(i).await.unwrap();
+                    }
+                });
+            }
+
+            for _ in 0..1_000 * 5 {
+                let _ = rx.recv().await;
+            }
+        })
+    });
+}
+
+fn contention_unbounded(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    b.iter(|| {
+        rt.block_on(async move {
+            let (tx, mut rx) = mpsc::unbounded_channel::<usize>();
+
+            for _ in 0..5 {
+                let tx = tx.clone();
+                tokio::spawn(async move {
+                    for i in 0..1000 {
+                        tx.send(i).unwrap();
+                    }
+                });
+            }
+
+            for _ in 0..1_000 * 5 {
+                let _ = rx.recv().await;
+            }
+        })
+    });
+}
+
+fn uncontented_bounded(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    b.iter(|| {
+        rt.block_on(async move {
+            let (mut tx, mut rx) = mpsc::channel::<usize>(1_000_000);
+
+            for i in 0..5000 {
+                tx.send(i).await.unwrap();
+            }
+
+            for _ in 0..5_000 {
+                let _ = rx.recv().await;
+            }
+        })
+    });
+}
+
+fn uncontented_unbounded(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    b.iter(|| {
+        rt.block_on(async move {
+            let (tx, mut rx) = mpsc::unbounded_channel::<usize>();
+
+            for i in 0..5000 {
+                tx.send(i).unwrap();
+            }
+
+            for _ in 0..5_000 {
+                let _ = rx.recv().await;
+            }
+        })
+    });
+}
+
+bencher::benchmark_group!(
+    create,
+    create_1_medium,
+    create_100_medium,
+    create_100_000_medium
+);
+
+bencher::benchmark_group!(send, send_medium, send_large);
+
+bencher::benchmark_group!(
+    contention,
+    contention_bounded,
+    contention_bounded_full,
+    contention_unbounded,
+    uncontented_bounded,
+    uncontented_unbounded
+);
+
+bencher::benchmark_main!(create, send, contention);

benches/scheduler.rs

@@ -0,0 +1,152 @@
+//! Benchmark implementation details of the theaded scheduler. These benches are
+//! intended to be used as a form of regression testing and not as a general
+//! purpose benchmark demonstrating real-world performance.
+
+use tokio::runtime::{self, Runtime};
+use tokio::sync::oneshot;
+
+use bencher::{benchmark_group, benchmark_main, Bencher};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering::Relaxed;
+use std::sync::{mpsc, Arc};
+
+fn spawn_many(b: &mut Bencher) {
+    const NUM_SPAWN: usize = 10_000;
+
+    let mut rt = rt();
+
+    let (tx, rx) = mpsc::sync_channel(1000);
+    let rem = Arc::new(AtomicUsize::new(0));
+
+    b.iter(|| {
+        rem.store(NUM_SPAWN, Relaxed);
+
+        rt.block_on(async {
+            for _ in 0..NUM_SPAWN {
+                let tx = tx.clone();
+                let rem = rem.clone();
+
+                tokio::spawn(async move {
+                    if 1 == rem.fetch_sub(1, Relaxed) {
+                        tx.send(()).unwrap();
+                    }
+                });
+            }
+
+            let _ = rx.recv().unwrap();
+        });
+    });
+}
+
+fn yield_many(b: &mut Bencher) {
+    const NUM_YIELD: usize = 1_000;
+    const TASKS: usize = 200;
+
+    let rt = rt();
+
+    let (tx, rx) = mpsc::sync_channel(TASKS);
+
+    b.iter(move || {
+        for _ in 0..TASKS {
+            let tx = tx.clone();
+
+            rt.spawn(async move {
+                for _ in 0..NUM_YIELD {
+                    tokio::task::yield_now().await;
+                }
+
+                tx.send(()).unwrap();
+            });
+        }
+
+        for _ in 0..TASKS {
+            let _ = rx.recv().unwrap();
+        }
+    });
+}
+
+fn ping_pong(b: &mut Bencher) {
+    const NUM_PINGS: usize = 1_000;
+
+    let mut rt = rt();
+
+    let (done_tx, done_rx) = mpsc::sync_channel(1000);
+    let rem = Arc::new(AtomicUsize::new(0));
+
+    b.iter(|| {
+        let done_tx = done_tx.clone();
+        let rem = rem.clone();
+        rem.store(NUM_PINGS, Relaxed);
+
+        rt.block_on(async {
+            tokio::spawn(async move {
+                for _ in 0..NUM_PINGS {
+                    let rem = rem.clone();
+                    let done_tx = done_tx.clone();
+
+                    tokio::spawn(async move {
+                        let (tx1, rx1) = oneshot::channel();
+                        let (tx2, rx2) = oneshot::channel();
+
+                        tokio::spawn(async move {
+                            rx1.await.unwrap();
+                            tx2.send(()).unwrap();
+                        });
+
+                        tx1.send(()).unwrap();
+                        rx2.await.unwrap();
+
+                        if 1 == rem.fetch_sub(1, Relaxed) {
+                            done_tx.send(()).unwrap();
+                        }
+                    });
+                }
+            });
+
+            done_rx.recv().unwrap();
+        });
+    });
+}
+
+fn chained_spawn(b: &mut Bencher) {
+    const ITER: usize = 1_000;
+
+    let mut rt = rt();
+
+    fn iter(done_tx: mpsc::SyncSender<()>, n: usize) {
+        if n == 0 {
+            done_tx.send(()).unwrap();
+        } else {
+            tokio::spawn(async move {
+                iter(done_tx, n - 1);
+            });
+        }
+    }
+
+    let (done_tx, done_rx) = mpsc::sync_channel(1000);
+
+    b.iter(move || {
+        let done_tx = done_tx.clone();
+
+        rt.block_on(async {
+            tokio::spawn(async move {
+                iter(done_tx, ITER);
+            });
+
+            done_rx.recv().unwrap();
+        });
+    });
+}
+
+fn rt() -> Runtime {
+    runtime::Builder::new()
+        .threaded_scheduler()
+        .core_threads(4)
+        .enable_all()
+        .build()
+        .unwrap()
+}
+
+benchmark_group!(scheduler, spawn_many, ping_pong, yield_many, chained_spawn,);
+
+benchmark_main!(scheduler);

benches/spawn.rs

@@ -0,0 +1,70 @@
+//! Benchmark spawning a task onto the basic and threaded Tokio executors.
+//! This essentially measure the time to enqueue a task in the local and remote
+//! case.
+
+use bencher::{black_box, Bencher};
+
+async fn work() -> usize {
+    let val = 1 + 1;
+    black_box(val)
+}
+
+fn basic_scheduler_local_spawn(bench: &mut Bencher) {
+    let mut runtime = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+    runtime.block_on(async {
+        bench.iter(|| {
+            let h = tokio::spawn(work());
+            black_box(h);
+        })
+    });
+}
+
+fn threaded_scheduler_local_spawn(bench: &mut Bencher) {
+    let mut runtime = tokio::runtime::Builder::new()
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+    runtime.block_on(async {
+        bench.iter(|| {
+            let h = tokio::spawn(work());
+            black_box(h);
+        })
+    });
+}
+
+fn basic_scheduler_remote_spawn(bench: &mut Bencher) {
+    let runtime = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+    let handle = runtime.handle();
+    bench.iter(|| {
+        let h = handle.spawn(work());
+        black_box(h);
+    });
+}
+
+fn threaded_scheduler_remote_spawn(bench: &mut Bencher) {
+    let runtime = tokio::runtime::Builder::new()
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+    let handle = runtime.handle();
+    bench.iter(|| {
+        let h = handle.spawn(work());
+        black_box(h);
+    });
+}
+
+bencher::benchmark_group!(
+    spawn,
+    basic_scheduler_local_spawn,
+    threaded_scheduler_local_spawn,
+    basic_scheduler_remote_spawn,
+    threaded_scheduler_remote_spawn
+);
+
+bencher::benchmark_main!(spawn);

benches/sync_rwlock.rs

@@ -0,0 +1,147 @@
+use bencher::{black_box, Bencher};
+use std::sync::Arc;
+use tokio::{sync::RwLock, task};
+
+fn read_uncontended(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    let lock = Arc::new(RwLock::new(()));
+    b.iter(|| {
+        let lock = lock.clone();
+        rt.block_on(async move {
+            for _ in 0..6 {
+                let read = lock.read().await;
+                black_box(read);
+            }
+        })
+    });
+}
+
+fn read_concurrent_uncontended_multi(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    async fn task(lock: Arc<RwLock<()>>) {
+        let read = lock.read().await;
+        black_box(read);
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    b.iter(|| {
+        let lock = lock.clone();
+        rt.block_on(async move {
+            let j = tokio::try_join! {
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone()))
+            };
+            j.unwrap();
+        })
+    });
+}
+
+fn read_concurrent_uncontended(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+
+    async fn task(lock: Arc<RwLock<()>>) {
+        let read = lock.read().await;
+        black_box(read);
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    b.iter(|| {
+        let lock = lock.clone();
+        rt.block_on(async move {
+            tokio::join! {
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone())
+            };
+        })
+    });
+}
+
+fn read_concurrent_contended_multi(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    async fn task(lock: Arc<RwLock<()>>) {
+        let read = lock.read().await;
+        black_box(read);
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    b.iter(|| {
+        let lock = lock.clone();
+        rt.block_on(async move {
+            let write = lock.write().await;
+            let j = tokio::try_join! {
+                async move { drop(write); Ok(()) },
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+                task::spawn(task(lock.clone())),
+            };
+            j.unwrap();
+        })
+    });
+}
+
+fn read_concurrent_contended(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+
+    async fn task(lock: Arc<RwLock<()>>) {
+        let read = lock.read().await;
+        black_box(read);
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    b.iter(|| {
+        let lock = lock.clone();
+        rt.block_on(async move {
+            let write = lock.write().await;
+            tokio::join! {
+                async move { drop(write) },
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+                task(lock.clone()),
+            };
+        })
+    });
+}
+
+bencher::benchmark_group!(
+    sync_rwlock,
+    read_uncontended,
+    read_concurrent_uncontended,
+    read_concurrent_uncontended_multi,
+    read_concurrent_contended,
+    read_concurrent_contended_multi
+);
+
+bencher::benchmark_main!(sync_rwlock);

benches/sync_semaphore.rs

@@ -0,0 +1,130 @@
+use bencher::Bencher;
+use std::sync::Arc;
+use tokio::{sync::Semaphore, task};
+
+fn uncontended(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    let s = Arc::new(Semaphore::new(10));
+    b.iter(|| {
+        let s = s.clone();
+        rt.block_on(async move {
+            for _ in 0..6 {
+                let permit = s.acquire().await;
+                drop(permit);
+            }
+        })
+    });
+}
+
+async fn task(s: Arc<Semaphore>) {
+    let permit = s.acquire().await;
+    drop(permit);
+}
+
+fn uncontended_concurrent_multi(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    let s = Arc::new(Semaphore::new(10));
+    b.iter(|| {
+        let s = s.clone();
+        rt.block_on(async move {
+            let j = tokio::try_join! {
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone()))
+            };
+            j.unwrap();
+        })
+    });
+}
+
+fn uncontended_concurrent_single(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+
+    let s = Arc::new(Semaphore::new(10));
+    b.iter(|| {
+        let s = s.clone();
+        rt.block_on(async move {
+            tokio::join! {
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone())
+            };
+        })
+    });
+}
+
+fn contended_concurrent_multi(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .core_threads(6)
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+
+    let s = Arc::new(Semaphore::new(5));
+    b.iter(|| {
+        let s = s.clone();
+        rt.block_on(async move {
+            let j = tokio::try_join! {
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone())),
+                task::spawn(task(s.clone()))
+            };
+            j.unwrap();
+        })
+    });
+}
+
+fn contended_concurrent_single(b: &mut Bencher) {
+    let mut rt = tokio::runtime::Builder::new()
+        .basic_scheduler()
+        .build()
+        .unwrap();
+
+    let s = Arc::new(Semaphore::new(5));
+    b.iter(|| {
+        let s = s.clone();
+        rt.block_on(async move {
+            tokio::join! {
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone()),
+                task(s.clone())
+            };
+        })
+    });
+}
+
+bencher::benchmark_group!(
+    sync_semaphore,
+    uncontended,
+    uncontended_concurrent_multi,
+    uncontended_concurrent_single,
+    contended_concurrent_multi,
+    contended_concurrent_single
+);
+
+bencher::benchmark_main!(sync_semaphore);

ci/azure-cargo-check.yml

@@ -1,29 +0,0 @@
-parameters:
-  noDefaultFeatures: '--no-default-features'
-
-jobs:
-- job: ${{ parameters.name }}
-  displayName: ${{ parameters.displayName }}
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-is-release.yml
-
-  - ${{ each crate in parameters.crates }}:
-    - ${{ each feature in crate.value }}:
-      - script: cargo check ${{ parameters.noDefaultFeatures }} --features ${{ feature }}
-        displayName: Check `${{ crate.key }}`, features = ${{ feature }}
-        workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}
-        condition: and(succeeded(), not(variables['isRelease']))
-
-  - template: azure-patch-crates.yml
-
-  - ${{ each crate in parameters.crates }}:
-    - ${{ each feature in crate.value }}:
-      - script: cargo check ${{ parameters.noDefaultFeatures }} --features ${{ feature }}
-        displayName: Check `${{ crate.key }}`, features = ${{ feature }}
-        workingDirectory: $(Build.SourcesDirectory)/${{ crate.key }}

ci/azure-check-docs.yml

@@ -1,15 +0,0 @@
-jobs:
-# Check docs
-- job: ${{ parameters.name }}
-  displayName: Check docs
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-    - template: azure-install-rust.yml
-      parameters:
-        rust_version: ${{ parameters.rust }}
-
-    - script: |
-        RUSTDOCFLAGS="--cfg docsrs" cargo doc --lib --no-deps --all-features
-      displayName: Check docs
-

ci/azure-check-features.yml

@@ -1,32 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: Check features
-  strategy:
-    matrix:
-      Linux:
-        vmImage: ubuntu-16.04
-      MacOS:
-        vmImage: macOS-10.13
-      Windows:
-        vmImage: vs2017-win2016
-  pool:
-    vmImage: $(vmImage)
-
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-patch-crates.yml
-
-  - script: cargo install cargo-hack
-    displayName: Install cargo-hack
-
-  # Check each feature works properly
-  # * --each-feature
-  #   run for each feature which includes --no-default-features and default features of package
-  # * -Z avoid-dev-deps
-  #   build without dev-dependencies to avoid https://github.com/rust-lang/cargo/issues/4866
-  #   tracking-issue: https://github.com/rust-lang/cargo/issues/5133
-  - script: cargo hack check --all --each-feature -Z avoid-dev-deps
-    displayName: cargo hack check --all --each-feature

ci/azure-check-minrust.yml

@@ -1,14 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: Min supported Rust version
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-patch-crates.yml
-
-  - script: cargo check --all
-    displayName: cargo check --all

ci/azure-clippy.yml

@@ -1,16 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: Clippy
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-    - template: azure-install-rust.yml
-      parameters:
-        rust_version: ${{ parameters.rust }}
-    - script: |
-        rustup component add clippy
-        cargo clippy --version
-      displayName: Install clippy
-    - script: |
-        cargo clippy --all --all-features -- -A clippy::mutex-atomic
-      displayName: cargo clippy --all

ci/azure-cross-compile.yml

@@ -1,44 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: ${{ parameters.displayName }}
-  strategy:
-    matrix:
-      i686:
-        vmImage: ubuntu-16.04
-        target: i686-unknown-linux-gnu
-      powerpc:
-        vmImage: ubuntu-16.04
-        target: powerpc-unknown-linux-gnu
-      powerpc64:
-        vmImage: ubuntu-16.04
-        target: powerpc64-unknown-linux-gnu
-      mips:
-        vmImage: ubuntu-16.04
-        target: mips-unknown-linux-gnu
-      arm:
-        vmImage: ubuntu-16.04
-        target: arm-linux-androideabi
-  pool:
-    vmImage: $(vmImage)
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - script: sudo apt-get update
-    displayName: apt-get update
-
-  - script: sudo apt-get install gcc-multilib
-    displayName: Install gcc-multilib
-
-  - script: cargo install cross
-    displayName: Install cross
-
-  # Always patch
-  - template: azure-patch-crates.yml
-
-  - script: cross check --all --exclude tokio-tls --target $(target)
-    displayName: Check source
-
-  # - script: cross check --tests --all --exclude tokio-tls --target $(target)
-  #   displayName: Check tests

ci/azure-deploy-docs.yml

@@ -1,39 +0,0 @@
-parameters:
-  dependsOn: []
-
-jobs:
-- job: documentation
-  displayName: 'Deploy API Documentation'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-  pool:
-    vmImage: 'Ubuntu 16.04'
-  dependsOn:
-    - ${{ parameters.dependsOn }}
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      # rust_version: stable
-      rust_version: ${{ parameters.rust }}
-  - script: |
-      cargo doc --all --no-deps --all-features
-      cp -R target/doc '$(Build.BinariesDirectory)'
-    displayName: 'Generate Documentation'
-  - script: |
-      set -e
-
-      git --version
-      ls -la
-      git init
-      git config user.name 'Deployment Bot (from Azure Pipelines)'
-      git config user.email '[email protected]'
-      git config --global credential.helper 'store --file ~/.my-credentials'
-      printf "protocol=https\nhost=github.com\nusername=carllerche\npassword=%s\n\n" "$GITHUB_TOKEN" | git credential-store --file ~/.my-credentials store
-      git remote add origin https://github.com/tokio-rs/tokio
-      git checkout -b gh-pages
-      git add .
-      git commit -m 'Deploy Tokio API documentation'
-      git push -f origin gh-pages
-    env:
-      GITHUB_TOKEN: $(githubPersonalToken)
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Deploy Documentation'

ci/azure-install-rust.yml

@@ -1,33 +0,0 @@
-steps:
-  # Linux and macOS.
-  - script: |
-      set -e
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain none
-      export PATH=$PATH:$HOME/.cargo/bin
-      rustup toolchain install $RUSTUP_TOOLCHAIN
-      rustup default $RUSTUP_TOOLCHAIN
-      echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin"
-    env:
-      RUSTUP_TOOLCHAIN: ${{parameters.rust_version}}
-    displayName: "Install rust (*nix)"
-    condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
-
-  # Windows.
-  - script: |
-      curl -sSf -o rustup-init.exe https://win.rustup.rs
-      rustup-init.exe -y --profile minimal --default-toolchain none
-      set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-      rustup toolchain install %RUSTUP_TOOLCHAIN%
-      rustup default %RUSTUP_TOOLCHAIN%
-      echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin"
-    env:
-      RUSTUP_TOOLCHAIN: ${{parameters.rust_version}}
-    displayName: "Install rust (windows)"
-    condition: eq(variables['Agent.OS'], 'Windows_NT')
-
-  # All platforms.
-  - script: |
-        rustup toolchain list
-        rustc -Vv
-        cargo -V
-    displayName: Query rust and cargo versions

ci/azure-is-release.yml

@@ -1,9 +0,0 @@
-steps:
-  - bash: |
-      set -e
-
-      if git log --no-merges -1 --format='%B' | grep -qF '[ci-release]'; then
-        echo "##vso[task.setvariable variable=isRelease]true"
-      fi
-    failOnStderr: true
-    displayName: Check if release commit

ci/azure-loom.yml

@@ -1,18 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: Loom tests
-  pool:
-    vmImage: ubuntu-16.04
-
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - ${{ each crate in parameters.crates }}:
-    - script: RUSTFLAGS="--cfg loom" cargo test --lib --release --features "full" -- --test-threads=1 --nocapture
-      env:
-        LOOM_MAX_PREEMPTIONS: 1
-        CI: 'True'
-      displayName: test ${{ crate }}
-      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}

ci/azure-patch-crates.yml

@@ -1,16 +0,0 @@
-steps:
-  - script: |
-      set -e
-
-      # Remove any existing patch statements
-      mv Cargo.toml Cargo.toml.bck
-      sed -n '/\[patch.crates-io\]/q;p' Cargo.toml.bck > Cargo.toml
-
-      # Patch all crates
-      cat ci/patch.toml >> Cargo.toml
-
-      # Print `Cargo.toml` for debugging
-      echo "~~~~ Cargo.toml ~~~~"
-      cat Cargo.toml
-      echo "~~~~~~~~~~~~~~~~~~~~"
-    displayName: Patch Cargo.toml

ci/azure-rustfmt.yml

@@ -1,17 +0,0 @@
-jobs:
-# Check formatting
-- job: ${{ parameters.name }}
-  displayName: Check rustfmt
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-    - template: azure-install-rust.yml
-      parameters:
-        rust_version: ${{ parameters.rust }}
-    - script: |
-        rustup component add rustfmt
-        cargo fmt --version
-      displayName: Install rustfmt
-    - script: |
-        cargo fmt --all -- --check
-      displayName: Check formatting

ci/azure-test-build.yml

@@ -1,17 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: ${{ parameters.displayName }}
-  pool:
-    vmImage: 'Ubuntu 16.04'
-
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - script: cargo install cargo-hack
-    displayName: Install cargo-hack
-
-  - script: cargo hack test --each-feature
-    displayName: cargo hack test --each-feature
-    workingDirectory: $(Build.SourcesDirectory)/tests-build

ci/azure-test-nightly.yml

@@ -1,19 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: ${{ parameters.displayName }}
-  pool:
-    vmImage: ubuntu-16.04
-
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-patch-crates.yml
-
-  - script: cargo check --all
-    displayName: cargo check --all
-
-  # Check benches
-  - script: cargo check --benches --all
-    displayName: Check benchmarks

ci/azure-test-stable.yml

@@ -1,42 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: ${{ parameters.displayName }}
-  strategy:
-    matrix:
-      Linux:
-        vmImage: ubuntu-16.04
-
-      ${{ if parameters.cross }}:
-        MacOS:
-          vmImage: macOS-10.13
-        Windows:
-          vmImage: vs2017-win2016
-  pool:
-    vmImage: $(vmImage)
-
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-is-release.yml
-
-  - ${{ each crate in parameters.crates }}:
-    # Run with all crate features
-    - script: cargo test --all-features
-      env:
-        LOOM_MAX_PREEMPTIONS: 2
-        CI: 'True'
-      displayName: ${{ crate }} - cargo test --all-features
-      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}
-
-  - template: azure-patch-crates.yml
-
-  - ${{ each crate in parameters.crates }}:
-    # Run with all crate features
-    - script: cargo test --all-features
-      env:
-        LOOM_MAX_PREEMPTIONS: 2
-        CI: 'True'
-      displayName: ${{ crate }} - cargo test --all-features
-      workingDirectory: $(Build.SourcesDirectory)/${{ crate }}

ci/azure-tsan.yml

@@ -1,34 +0,0 @@
-jobs:
-- job: ${{ parameters.name }}
-  displayName: TSAN
-  strategy:
-    matrix:
-      Timer:
-        cmd: cargo test -p tokio-timer --test hammer
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-  - template: azure-install-rust.yml
-    parameters:
-      rust_version: ${{ parameters.rust }}
-
-  - template: azure-patch-crates.yml
-  - script: |
-        set -e
-
-        # Make sure the benchmarks compile
-        export ASAN_OPTIONS="detect_odr_violation=0 detect_leaks=0"
-        export TSAN_OPTIONS="suppressions=`pwd`/ci/tsan"
-        export RUST_BACKTRACE=1
-
-        # Run address sanitizer
-        RUSTFLAGS="-Z sanitizer=address" \
-        $(cmd) --target x86_64-unknown-linux-gnu
-
-        # Run thread sanitizer
-        RUSTFLAGS="-Z sanitizer=thread" \
-        $(cmd) --target x86_64-unknown-linux-gnu
-    displayName: TSAN / MSAN
-    env:
-      TSAN: yes
-

ci/patch.toml

@@ -1,8 +0,0 @@
-# Patch dependencies to run all tests against versions of the crate in the
-# repository.
-[patch.crates-io]
-tokio = { path = "tokio" }
-tokio-macros = { path = "tokio-macros" }
-tokio-test = { path = "tokio-test" }
-tokio-tls = { path = "tokio-tls" }
-tokio-util = { path = "tokio-util" }

ci/tsan

@@ -1,39 +0,0 @@
-# TSAN suppressions file for Tokio
-
-# TSAN does not understand fences and `Arc::drop` is implemented using a fence.
-# This causes many false positives.
-race:Arc*drop
-race:Weak*drop
-
-# `std` mpsc is not used in any Tokio code base. This race is triggered by some
-# rust runtime logic.
-race:std*mpsc_queue
-race:std*lang_start
-race:drop*std::thread*
-
-# Probably more fences in std.
-race:__call_tls_dtors
-
-# The epoch-based GC uses fences.
-race:crossbeam_epoch
-
-# Push and steal operations in crossbeam-deque may cause data races, but such
-# data races are safe. If a data race happens, the value read by `steal` is
-# forgotten and the steal operation is then retried.
-race:crossbeam_deque*push
-race:crossbeam_deque*steal
-
-# This filters out expected data race in the Treiber stack implementations.
-# Treiber stacks are inherently racy. The pop operation will attempt to access
-# the "next" pointer on the node it is attempting to pop. However, at this
-# point it has not gained ownership of the node and another thread might beat
-# it and take ownership of the node first (touching the next pointer). The
-# original pop operation will fail due to the ABA guard, but tsan still picks
-# up the access on the next pointer.
-race:Backup::next_sleeper
-race:Backup::set_next_sleeper
-race:WorkerEntry::set_next_sleeper
-
-# This ignores a false positive caused by `thread::park()`/`thread::unpark()`.
-# See: https://github.com/rust-lang/rust/pull/54806#issuecomment-436193353
-race:pthread_cond_destroy

examples/Cargo.toml

@@ -4,9 +4,13 @@ version = "0.0.0"
 publish = false
 edition = "2018"
 
+# If you copy one of the examples into a new project, you should be using
+# [dependencies] instead.
 [dev-dependencies]
-tokio = { version = "0.2.0", path = "../tokio", features = ["full"] }
-tokio-util = { version = "0.2.0", path = "../tokio-util", features = ["full"] }
+tokio = { version = "0.2.0", path = "../tokio", features = ["full", "tracing"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.2.7", default-features = false, features = ["fmt", "ansi", "env-filter", "chrono", "tracing-log"] }
+tokio-util = { version = "0.3.0", path = "../tokio-util", features = ["full"] }
 bytes = "0.5"
 futures = "0.3.0"
 http = "0.2"

examples/README.md

@@ -1,6 +1,23 @@
 ## Examples of how to use Tokio
 
-The `master` branch is currently being updated to use `async` / `await`.
-The examples are not fully ported. Examples for stable Tokio can be
-found
-[here](https://github.com/tokio-rs/tokio/tree/v0.1.x/tokio/examples).
+This directory contains a number of examples showcasing various capabilities of
+the `tokio` crate.
+
+All examples can be executed with:
+
+```
+cargo run --example $name
+```
+
+A good starting point for the examples would be [`hello_world`](hello_world.rs)
+and [`echo`](echo.rs). Additionally [the tokio website][tokioweb] contains
+additional guides for some of the examples.
+
+For a larger "real world" example, see the [`mini-redis`][redis] repository.
+
+If you've got an example you'd like to see here, please feel free to open an
+issue. Otherwise if you've got an example you'd like to add, please feel free
+to make a PR!
+
+[tokioweb]: https://tokio.rs/tokio/tutorial
+[redis]: https://github.com/tokio-rs/mini-redis

examples/chat.rs

@@ -43,6 +43,26 @@ use std::task::{Context, Poll};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn Error>> {
+    use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter};
+    // Configure a `tracing` subscriber that logs traces emitted by the chat
+    // server.
+    tracing_subscriber::fmt()
+        // Filter what traces are displayed based on the RUST_LOG environment
+        // variable.
+        //
+        // Traces emitted by the example code will always be displayed. You
+        // can set `RUST_LOG=tokio=trace` to enable additional traces emitted by
+        // Tokio itself.
+        .with_env_filter(EnvFilter::from_default_env().add_directive("chat=info".parse()?))
+        // Log events when `tracing` spans are created, entered, exited, or
+        // closed. When Tokio's internal tracing support is enabled (as
+        // described above), this can be used to track the lifecycle of spawned
+        // tasks on the Tokio runtime.
+        .with_span_events(FmtSpan::FULL)
+        // Set this subscriber as the default, to collect all traces emitted by
+        // the program.
+        .init();
+
     // Create the shared state. This is how all the peers communicate.
     //
     // The server task will hold a handle to this. For every new client, the
@@ -59,7 +79,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     // Note that this is the Tokio TcpListener, which is fully async.
     let mut listener = TcpListener::bind(&addr).await?;
 
-    println!("server running on {}", addr);
+    tracing::info!("server running on {}", addr);
 
     loop {
         // Asynchronously wait for an inbound TcpStream.
@@ -70,8 +90,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
         // Spawn our handler to be run asynchronously.
         tokio::spawn(async move {
+            tracing::debug!("accepted connection");
             if let Err(e) = process(state, stream, addr).await {
-                println!("an error occured; error = {:?}", e);
+                tracing::info!("an error occurred; error = {:?}", e);
             }
         });
     }
@@ -175,7 +196,7 @@ impl Stream for Peer {
             // We've received a message we should broadcast to others.
             Some(Ok(message)) => Some(Ok(Message::Broadcast(message))),
 
-            // An error occured.
+            // An error occurred.
             Some(Err(e)) => Some(Err(e)),
 
             // The stream has been exhausted.
@@ -193,16 +214,14 @@ async fn process(
     let mut lines = Framed::new(stream, LinesCodec::new());
 
     // Send a prompt to the client to enter their username.
-    lines
-        .send(String::from("Please enter your username:"))
-        .await?;
+    lines.send("Please enter your username:").await?;
 
     // Read the first line from the `LineCodec` stream to get the username.
     let username = match lines.next().await {
         Some(Ok(line)) => line,
         // We didn't get a line so we return early here.
         _ => {
-            println!("Failed to get username from {}. Client disconnected.", addr);
+            tracing::error!("Failed to get username from {}. Client disconnected.", addr);
             return Ok(());
         }
     };
@@ -214,7 +233,7 @@ async fn process(
     {
         let mut state = state.lock().await;
         let msg = format!("{} has joined the chat", username);
-        println!("{}", msg);
+        tracing::info!("{}", msg);
         state.broadcast(addr, &msg).await;
     }
 
@@ -232,12 +251,13 @@ async fn process(
             // A message was received from a peer. Send it to the
             // current user.
             Ok(Message::Received(msg)) => {
-                peer.lines.send(msg).await?;
+                peer.lines.send(&msg).await?;
             }
             Err(e) => {
-                println!(
-                    "an error occured while processing messages for {}; error = {:?}",
-                    username, e
+                tracing::error!(
+                    "an error occurred while processing messages for {}; error = {:?}",
+                    username,
+                    e
                 );
             }
         }
@@ -250,7 +270,7 @@ async fn process(
         state.peers.remove(&addr);
 
         let msg = format!("{} has left the chat", username);
-        println!("{}", msg);
+        tracing::info!("{}", msg);
         state.broadcast(addr, &msg).await;
     }
 

examples/connect.rs

@@ -16,8 +16,9 @@
 
 #![warn(rust_2018_idioms)]
 
+use futures::StreamExt;
 use tokio::io;
-use tokio_util::codec::{FramedRead, FramedWrite};
+use tokio_util::codec::{BytesCodec, FramedRead, FramedWrite};
 
 use std::env;
 use std::error::Error;
@@ -41,8 +42,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .ok_or("this program requires at least one argument")?;
     let addr = addr.parse::<SocketAddr>()?;
 
-    let stdin = FramedRead::new(io::stdin(), codec::Bytes);
-    let stdout = FramedWrite::new(io::stdout(), codec::Bytes);
+    let stdin = FramedRead::new(io::stdin(), BytesCodec::new());
+    let stdin = stdin.map(|i| i.map(|bytes| bytes.freeze()));
+    let stdout = FramedWrite::new(io::stdout(), BytesCodec::new());
 
     if tcp {
         tcp::connect(&addr, stdin, stdout).await?;
@@ -54,25 +56,26 @@ async fn main() -> Result<(), Box<dyn Error>> {
 }
 
 mod tcp {
-    use super::codec;
-    use futures::StreamExt;
-    use futures::{future, Sink, SinkExt};
+    use bytes::Bytes;
+    use futures::{future, Sink, SinkExt, Stream, StreamExt};
     use std::{error::Error, io, net::SocketAddr};
     use tokio::net::TcpStream;
-    use tokio::stream::Stream;
-    use tokio_util::codec::{FramedRead, FramedWrite};
+    use tokio_util::codec::{BytesCodec, FramedRead, FramedWrite};
 
     pub async fn connect(
         addr: &SocketAddr,
-        mut stdin: impl Stream<Item = Result<Vec<u8>, io::Error>> + Unpin,
-        mut stdout: impl Sink<Vec<u8>, Error = io::Error> + Unpin,
+        mut stdin: impl Stream<Item = Result<Bytes, io::Error>> + Unpin,
+        mut stdout: impl Sink<Bytes, Error = io::Error> + Unpin,
     ) -> Result<(), Box<dyn Error>> {
         let mut stream = TcpStream::connect(addr).await?;
         let (r, w) = stream.split();
-        let mut sink = FramedWrite::new(w, codec::Bytes);
-        let mut stream = FramedRead::new(r, codec::Bytes)
+        let mut sink = FramedWrite::new(w, BytesCodec::new());
+        // filter map Result<BytesMut, Error> stream into just a Bytes stream to match stdout Sink
+        // on the event of an Error, log the error and end the stream
+        let mut stream = FramedRead::new(r, BytesCodec::new())
             .filter_map(|i| match i {
-                Ok(i) => future::ready(Some(i)),
+                //BytesMut into Bytes
+                Ok(i) => future::ready(Some(i.freeze())),
                 Err(e) => {
                     println!("failed to read from socket; error={}", e);
                     future::ready(None)
@@ -88,19 +91,18 @@ mod tcp {
 }
 
 mod udp {
-    use tokio::net::udp::{RecvHalf, SendHalf};
-    use tokio::net::UdpSocket;
-    use tokio::stream::{Stream, StreamExt};
-
-    use futures::{future, Sink, SinkExt};
+    use bytes::Bytes;
+    use futures::{future, Sink, SinkExt, Stream, StreamExt};
     use std::error::Error;
     use std::io;
     use std::net::SocketAddr;
+    use tokio::net::udp::{RecvHalf, SendHalf};
+    use tokio::net::UdpSocket;
 
     pub async fn connect(
         addr: &SocketAddr,
-        stdin: impl Stream<Item = Result<Vec<u8>, io::Error>> + Unpin,
-        stdout: impl Sink<Vec<u8>, Error = io::Error> + Unpin,
+        stdin: impl Stream<Item = Result<Bytes, io::Error>> + Unpin,
+        stdout: impl Sink<Bytes, Error = io::Error> + Unpin,
     ) -> Result<(), Box<dyn Error>> {
         // We'll bind our UDP socket to a local IP/port, but for now we
         // basically let the OS pick both of those.
@@ -120,7 +122,7 @@ mod udp {
     }
 
     async fn send(
-        mut stdin: impl Stream<Item = Result<Vec<u8>, io::Error>> + Unpin,
+        mut stdin: impl Stream<Item = Result<Bytes, io::Error>> + Unpin,
         writer: &mut SendHalf,
     ) -> Result<(), io::Error> {
         while let Some(item) = stdin.next().await {
@@ -132,7 +134,7 @@ mod udp {
     }
 
     async fn recv(
-        mut stdout: impl Sink<Vec<u8>, Error = io::Error> + Unpin,
+        mut stdout: impl Sink<Bytes, Error = io::Error> + Unpin,
         reader: &mut RecvHalf,
     ) -> Result<(), io::Error> {
         loop {
@@ -140,47 +142,8 @@ mod udp {
             let n = reader.recv(&mut buf[..]).await?;
 
             if n > 0 {
-                stdout.send(buf).await?;
+                stdout.send(Bytes::from(buf)).await?;
             }
         }
     }
 }
-
-mod codec {
-    use bytes::{BufMut, BytesMut};
-    use std::io;
-    use tokio_util::codec::{Decoder, Encoder};
-
-    /// A simple `Codec` implementation that just ships bytes around.
-    ///
-    /// This type is used for "framing" a TCP/UDP stream of bytes but it's really
-    /// just a convenient method for us to work with streams/sinks for now.
-    /// This'll just take any data read and interpret it as a "frame" and
-    /// conversely just shove data into the output location without looking at
-    /// it.
-    pub struct Bytes;
-
-    impl Decoder for Bytes {
-        type Item = Vec<u8>;
-        type Error = io::Error;
-
-        fn decode(&mut self, buf: &mut BytesMut) -> io::Result<Option<Vec<u8>>> {
-            if !buf.is_empty() {
-                let len = buf.len();
-                Ok(Some(buf.split_to(len).into_iter().collect()))
-            } else {
-                Ok(None)
-            }
-        }
-    }
-
-    impl Encoder for Bytes {
-        type Item = Vec<u8>;
-        type Error = io::Error;
-
-        fn encode(&mut self, data: Vec<u8>, buf: &mut BytesMut) -> io::Result<()> {
-            buf.put(&data[..]);
-            Ok(())
-        }
-    }
-}

examples/echo-udp.rs

@@ -15,7 +15,6 @@
 use std::error::Error;
 use std::net::SocketAddr;
 use std::{env, io};
-use tokio;
 use tokio::net::UdpSocket;
 
 struct Server {

examples/echo.rs

@@ -21,7 +21,6 @@
 
 #![warn(rust_2018_idioms)]
 
-use tokio;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpListener;
 

examples/proxy.rs

@@ -23,6 +23,7 @@
 #![warn(rust_2018_idioms)]
 
 use tokio::io;
+use tokio::io::AsyncWriteExt;
 use tokio::net::{TcpListener, TcpStream};
 
 use futures::future::try_join;
@@ -63,8 +64,15 @@ async fn transfer(mut inbound: TcpStream, proxy_addr: String) -> Result<(), Box<
     let (mut ri, mut wi) = inbound.split();
     let (mut ro, mut wo) = outbound.split();
 
-    let client_to_server = io::copy(&mut ri, &mut wo);
-    let server_to_client = io::copy(&mut ro, &mut wi);
+    let client_to_server = async {
+        io::copy(&mut ri, &mut wo).await?;
+        wo.shutdown().await
+    };
+
+    let server_to_client = async {
+        io::copy(&mut ro, &mut wi).await?;
+        wi.shutdown().await
+    };
 
     try_join(client_to_server, server_to_client).await?;
 

examples/tinydb.rs

@@ -130,7 +130,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
                                 let response = response.serialize();
 
-                                if let Err(e) = lines.send(response).await {
+                                if let Err(e) = lines.send(response.as_str()).await {
                                     println!("error on sending response; error = {:?}", e);
                                 }
                             }

examples/tinyhttp.rs

@@ -18,7 +18,6 @@ use futures::SinkExt;
 use http::{header::HeaderValue, Request, Response, StatusCode};
 #[macro_use]
 extern crate serde_derive;
-use serde_json;
 use std::{env, error::Error, fmt, io};
 use tokio::net::{TcpListener, TcpStream};
 use tokio::stream::StreamExt;
@@ -96,8 +95,7 @@ struct Http;
 
 /// Implementation of encoding an HTTP response into a `BytesMut`, basically
 /// just writing out an HTTP/1.1 response.
-impl Encoder for Http {
-    type Item = Response<String>;
+impl Encoder<Response<String>> for Http {
     type Error = io::Error;
 
     fn encode(&mut self, item: Response<String>, dst: &mut BytesMut) -> io::Result<()> {

examples/udp-codec.rs

@@ -46,7 +46,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     // Run both futures simultaneously of `a` and `b` sending messages back and forth.
     match futures::future::try_join(a, b).await {
-        Err(e) => println!("an error occured; error = {:?}", e),
+        Err(e) => println!("an error occurred; error = {:?}", e),
         _ => println!("done!"),
     }
 

rustfmt.toml

@@ -1 +0,0 @@
-edition = "2018"

tests-integration/Cargo.toml

@@ -5,11 +5,24 @@ authors = ["Tokio Contributors <[email protected]>"]
 edition = "2018"
 publish = false
 
+[features]
+full = [
+    "macros",
+    "rt-core",
+    "rt-threaded",
+
+    "tokio/full",
+    "tokio-test"
+]
+macros = ["tokio/macros"]
+sync = ["tokio/sync"]
+rt-core = ["tokio/rt-core"]
+rt-threaded = ["rt-core", "tokio/rt-threaded"]
+
 [dependencies]
-tokio = { path = "../tokio", features = ["full"] }
+tokio = { path = "../tokio" }
+tokio-test = { path = "../tokio-test", optional = true }
 doc-comment = "0.3.1"
 
 [dev-dependencies]
-tokio-test = { path = "../tokio-test" }
-
 futures = { version = "0.3.0", features = ["async-await"] }

tests-integration/src/lib.rs

@@ -1,4 +1,2 @@
-use doc_comment::doc_comment;
-
-// #[doc = include_str!("../../README.md")]
-doc_comment!(include_str!("../../README.md"));
+#[cfg(feature = "full")]
+doc_comment::doc_comment!(include_str!("../../README.md"));

tests-integration/tests/macros_main.rs

@@ -0,0 +1,31 @@
+#![cfg(feature = "macros")]
+
+#[tokio::main]
+async fn basic_main() -> usize {
+    1
+}
+
+#[tokio::main]
+async fn generic_fun<T: Default>() -> T {
+    T::default()
+}
+
+#[cfg(feature = "rt-core")]
+mod spawn {
+    #[tokio::main]
+    async fn spawning() -> usize {
+        let join = tokio::spawn(async { 1 });
+        join.await.unwrap()
+    }
+
+    #[test]
+    fn main_with_spawn() {
+        assert_eq!(1, spawning());
+    }
+}
+
+#[test]
+fn shell() {
+    assert_eq!(1, basic_main());
+    assert_eq!(bool::default(), generic_fun::<bool>())
+}

tests-integration/tests/macros_pin.rs

@@ -0,0 +1,12 @@
+use futures::executor::block_on;
+
+async fn my_async_fn() {}
+
+#[test]
+fn pin() {
+    block_on(async {
+        let future = my_async_fn();
+        tokio::pin!(future);
+        (&mut future).await
+    });
+}

tests-integration/tests/macros_select.rs

@@ -0,0 +1,33 @@
+#![cfg(feature = "macros")]
+
+use futures::channel::oneshot;
+use futures::executor::block_on;
+use std::thread;
+
+#[test]
+fn join_with_select() {
+    block_on(async {
+        let (tx1, mut rx1) = oneshot::channel::<i32>();
+        let (tx2, mut rx2) = oneshot::channel::<i32>();
+
+        thread::spawn(move || {
+            tx1.send(123).unwrap();
+            tx2.send(456).unwrap();
+        });
+
+        let mut a = None;
+        let mut b = None;
+
+        while a.is_none() || b.is_none() {
+            tokio::select! {
+                v1 = (&mut rx1), if a.is_none() => a = Some(v1.unwrap()),
+                v2 = (&mut rx2), if b.is_none() => b = Some(v2.unwrap()),
+            }
+        }
+
+        let (a, b) = (a.unwrap(), b.unwrap());
+
+        assert_eq!(a, 123);
+        assert_eq!(b, 456);
+    });
+}

tests-integration/tests/process_stdio.rs

@@ -1,4 +1,5 @@
 #![warn(rust_2018_idioms)]
+#![cfg(feature = "full")]
 
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::process::{Child, Command};
@@ -25,8 +26,8 @@ fn cat() -> Command {
 }
 
 async fn feed_cat(mut cat: Child, n: usize) -> io::Result<ExitStatus> {
-    let mut stdin = cat.stdin().take().unwrap();
-    let stdout = cat.stdout().take().unwrap();
+    let mut stdin = cat.stdin.take().unwrap();
+    let stdout = cat.stdout.take().unwrap();
 
     // Produce n lines on the child's stdout.
     let write = async {
@@ -97,7 +98,7 @@ async fn feed_a_lot() {
 #[tokio::test]
 async fn wait_with_output_captures() {
     let mut child = cat().spawn().unwrap();
-    let mut stdin = child.stdin().take().unwrap();
+    let mut stdin = child.stdin.take().unwrap();
 
     let write_bytes = b"1234";
 

tests-integration/tests/rt_shell.rs

@@ -0,0 +1,32 @@
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "sync")]
+
+use tokio::runtime;
+use tokio::sync::oneshot;
+
+use std::sync::mpsc;
+use std::thread;
+
+#[test]
+fn basic_shell_rt() {
+    let (feed_tx, feed_rx) = mpsc::channel::<oneshot::Sender<()>>();
+
+    let th = thread::spawn(move || {
+        for tx in feed_rx.iter() {
+            tx.send(()).unwrap();
+        }
+    });
+
+    for _ in 0..1_000 {
+        let mut rt = runtime::Builder::new().build().unwrap();
+
+        let (tx, rx) = oneshot::channel();
+
+        feed_tx.send(tx).unwrap();
+
+        rt.block_on(rx).unwrap();
+    }
+
+    drop(feed_tx);
+    th.join().unwrap();
+}

tokio-macros/CHANGELOG.md

@@ -1,8 +1,38 @@
+# 0.2.5 (February 27, 2019)
+
+### Fixed
+- doc improvements ([#2225]).
+
+# 0.2.4 (January 27, 2019)
+
+### Fixed
+- generics on `#[tokio::main]` function ([#2177]).
+
+### Added
+- support for `tokio::select!` ([#2152]).
+
+# 0.2.3 (January 7, 2019)
+
+### Fixed
+- Revert breaking change.
+
+# 0.2.2 (January 7, 2019)
+
+### Added
+- General refactoring and inclusion of additional runtime options ([#2022] and [#2038])
+
 # 0.2.1 (December 18, 2019)
 
 ### Fixes
-- inherit visibility when wrapping async fn (#1954).
+- inherit visibility when wrapping async fn ([#1954]).
 
 # 0.2.0 (November 26, 2019)
 
 - Initial release
+
+[#2225]: https://github.com/tokio-rs/tokio/pull/2225
+[#2177]: https://github.com/tokio-rs/tokio/pull/2177
+[#2152]: https://github.com/tokio-rs/tokio/pull/2152
+[#2038]: https://github.com/tokio-rs/tokio/pull/2038
+[#2022]: https://github.com/tokio-rs/tokio/pull/2022
+[#1954]: https://github.com/tokio-rs/tokio/pull/1954

tokio-macros/Cargo.toml

@@ -7,13 +7,13 @@ name = "tokio-macros"
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "v0.1.x" git tag.
-version = "0.2.1"
+version = "0.2.5"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-macros/0.2.1/tokio_macros"
+documentation = "https://docs.rs/tokio-macros/0.2.5/tokio_macros"
 description = """
 Tokio's proc macros.
 """
@@ -25,6 +25,7 @@ proc-macro = true
 [features]
 
 [dependencies]
+proc-macro2 = "1.0.7"
 quote = "1"
 syn = { version = "1.0.3", features = ["full"] }
 

tokio-macros/src/entry.rs

@@ -0,0 +1,359 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use std::num::NonZeroUsize;
+
+#[derive(Clone, Copy, PartialEq)]
+enum Runtime {
+    Basic,
+    Threaded,
+}
+
+fn parse_knobs(
+    mut input: syn::ItemFn,
+    args: syn::AttributeArgs,
+    is_test: bool,
+    rt_threaded: bool,
+) -> Result<TokenStream, syn::Error> {
+    let sig = &mut input.sig;
+    let body = &input.block;
+    let attrs = &input.attrs;
+    let vis = input.vis;
+
+    if sig.asyncness.is_none() {
+        let msg = "the async keyword is missing from the function declaration";
+        return Err(syn::Error::new_spanned(sig.fn_token, msg));
+    }
+
+    sig.asyncness = None;
+
+    let mut runtime = None;
+    let mut core_threads = None;
+    let mut max_threads = None;
+
+    for arg in args {
+        match arg {
+            syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) => {
+                let ident = namevalue.path.get_ident();
+                if ident.is_none() {
+                    let msg = "Must have specified ident";
+                    return Err(syn::Error::new_spanned(namevalue, msg));
+                }
+                match ident.unwrap().to_string().to_lowercase().as_str() {
+                    "core_threads" => {
+                        if rt_threaded {
+                            match &namevalue.lit {
+                                syn::Lit::Int(expr) => {
+                                    let num = expr.base10_parse::<NonZeroUsize>().unwrap();
+                                    if num.get() > 1 {
+                                        runtime = Some(Runtime::Threaded);
+                                    } else {
+                                        runtime = Some(Runtime::Basic);
+                                    }
+
+                                    if let Some(v) = max_threads {
+                                        if v < num {
+                                            return Err(syn::Error::new_spanned(
+                                                namevalue,
+                                                "max_threads cannot be less than core_threads",
+                                            ));
+                                        }
+                                    }
+
+                                    core_threads = Some(num);
+                                }
+                                _ => {
+                                    return Err(syn::Error::new_spanned(
+                                        namevalue,
+                                        "core_threads argument must be an int",
+                                    ))
+                                }
+                            }
+                        } else {
+                            return Err(syn::Error::new_spanned(
+                                namevalue,
+                                "core_threads can only be set with rt-threaded feature flag enabled",
+                            ));
+                        }
+                    }
+                    "max_threads" => match &namevalue.lit {
+                        syn::Lit::Int(expr) => {
+                            let num = expr.base10_parse::<NonZeroUsize>().unwrap();
+
+                            if let Some(v) = core_threads {
+                                if num < v {
+                                    return Err(syn::Error::new_spanned(
+                                        namevalue,
+                                        "max_threads cannot be less than core_threads",
+                                    ));
+                                }
+                            }
+                            max_threads = Some(num);
+                        }
+                        _ => {
+                            return Err(syn::Error::new_spanned(
+                                namevalue,
+                                "max_threads argument must be an int",
+                            ))
+                        }
+                    },
+                    name => {
+                        let msg = format!("Unknown attribute pair {} is specified; expected one of: `core_threads`, `max_threads`", name);
+                        return Err(syn::Error::new_spanned(namevalue, msg));
+                    }
+                }
+            }
+            syn::NestedMeta::Meta(syn::Meta::Path(path)) => {
+                let ident = path.get_ident();
+                if ident.is_none() {
+                    let msg = "Must have specified ident";
+                    return Err(syn::Error::new_spanned(path, msg));
+                }
+                match ident.unwrap().to_string().to_lowercase().as_str() {
+                    "threaded_scheduler" => {
+                        runtime = Some(runtime.unwrap_or_else(|| Runtime::Threaded))
+                    }
+                    "basic_scheduler" => runtime = Some(runtime.unwrap_or_else(|| Runtime::Basic)),
+                    name => {
+                        let msg = format!("Unknown attribute {} is specified; expected `basic_scheduler` or `threaded_scheduler`", name);
+                        return Err(syn::Error::new_spanned(path, msg));
+                    }
+                }
+            }
+            other => {
+                return Err(syn::Error::new_spanned(
+                    other,
+                    "Unknown attribute inside the macro",
+                ));
+            }
+        }
+    }
+
+    let mut rt = quote! { tokio::runtime::Builder::new().basic_scheduler() };
+    if rt_threaded && (runtime == Some(Runtime::Threaded) || (runtime.is_none() && !is_test)) {
+        rt = quote! { #rt.threaded_scheduler() };
+    }
+    if let Some(v) = core_threads.map(|v| v.get()) {
+        rt = quote! { #rt.core_threads(#v) };
+    }
+    if let Some(v) = max_threads.map(|v| v.get()) {
+        rt = quote! { #rt.max_threads(#v) };
+    }
+
+    let header = {
+        if is_test {
+            quote! {
+                #[::core::prelude::v1::test]
+            }
+        } else {
+            quote! {}
+        }
+    };
+
+    let result = quote! {
+        #header
+        #(#attrs)*
+        #vis #sig {
+            #rt
+                .enable_all()
+                .build()
+                .unwrap()
+                .block_on(async { #body })
+        }
+    };
+
+    Ok(result.into())
+}
+
+#[cfg(not(test))] // Work around for rust-lang/rust#62127
+pub(crate) fn main(args: TokenStream, item: TokenStream, rt_threaded: bool) -> TokenStream {
+    let input = syn::parse_macro_input!(item as syn::ItemFn);
+    let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+    if input.sig.ident == "main" && !input.sig.inputs.is_empty() {
+        let msg = "the main function cannot accept arguments";
+        return syn::Error::new_spanned(&input.sig.inputs, msg)
+            .to_compile_error()
+            .into();
+    }
+
+    parse_knobs(input, args, false, rt_threaded).unwrap_or_else(|e| e.to_compile_error().into())
+}
+
+pub(crate) fn test(args: TokenStream, item: TokenStream, rt_threaded: bool) -> TokenStream {
+    let input = syn::parse_macro_input!(item as syn::ItemFn);
+    let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+    for attr in &input.attrs {
+        if attr.path.is_ident("test") {
+            let msg = "second test attribute is supplied";
+            return syn::Error::new_spanned(&attr, msg)
+                .to_compile_error()
+                .into();
+        }
+    }
+
+    if !input.sig.inputs.is_empty() {
+        let msg = "the test function cannot accept arguments";
+        return syn::Error::new_spanned(&input.sig.inputs, msg)
+            .to_compile_error()
+            .into();
+    }
+
+    parse_knobs(input, args, true, rt_threaded).unwrap_or_else(|e| e.to_compile_error().into())
+}
+
+pub(crate) mod old {
+    use proc_macro::TokenStream;
+    use quote::quote;
+
+    enum Runtime {
+        Basic,
+        Threaded,
+        Auto,
+    }
+
+    #[cfg(not(test))] // Work around for rust-lang/rust#62127
+    pub(crate) fn main(args: TokenStream, item: TokenStream) -> TokenStream {
+        let mut input = syn::parse_macro_input!(item as syn::ItemFn);
+        let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+        let sig = &mut input.sig;
+        let name = &sig.ident;
+        let inputs = &sig.inputs;
+        let body = &input.block;
+        let attrs = &input.attrs;
+        let vis = input.vis;
+
+        if sig.asyncness.is_none() {
+            let msg = "the async keyword is missing from the function declaration";
+            return syn::Error::new_spanned(sig.fn_token, msg)
+                .to_compile_error()
+                .into();
+        } else if name == "main" && !inputs.is_empty() {
+            let msg = "the main function cannot accept arguments";
+            return syn::Error::new_spanned(&sig.inputs, msg)
+                .to_compile_error()
+                .into();
+        }
+
+        sig.asyncness = None;
+
+        let mut runtime = Runtime::Auto;
+
+        for arg in args {
+            if let syn::NestedMeta::Meta(syn::Meta::Path(path)) = arg {
+                let ident = path.get_ident();
+                if ident.is_none() {
+                    let msg = "Must have specified ident";
+                    return syn::Error::new_spanned(path, msg).to_compile_error().into();
+                }
+                match ident.unwrap().to_string().to_lowercase().as_str() {
+                    "threaded_scheduler" => runtime = Runtime::Threaded,
+                    "basic_scheduler" => runtime = Runtime::Basic,
+                    name => {
+                        let msg = format!("Unknown attribute {} is specified; expected `basic_scheduler` or `threaded_scheduler`", name);
+                        return syn::Error::new_spanned(path, msg).to_compile_error().into();
+                    }
+                }
+            }
+        }
+
+        let result = match runtime {
+            Runtime::Threaded | Runtime::Auto => quote! {
+                #(#attrs)*
+                #vis #sig {
+                    tokio::runtime::Runtime::new().unwrap().block_on(async { #body })
+                }
+            },
+            Runtime::Basic => quote! {
+                #(#attrs)*
+                #vis #sig {
+                    tokio::runtime::Builder::new()
+                        .basic_scheduler()
+                        .enable_all()
+                        .build()
+                        .unwrap()
+                        .block_on(async { #body })
+                }
+            },
+        };
+
+        result.into()
+    }
+
+    pub(crate) fn test(args: TokenStream, item: TokenStream) -> TokenStream {
+        let input = syn::parse_macro_input!(item as syn::ItemFn);
+        let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+        let ret = &input.sig.output;
+        let name = &input.sig.ident;
+        let body = &input.block;
+        let attrs = &input.attrs;
+        let vis = input.vis;
+
+        for attr in attrs {
+            if attr.path.is_ident("test") {
+                let msg = "second test attribute is supplied";
+                return syn::Error::new_spanned(&attr, msg)
+                    .to_compile_error()
+                    .into();
+            }
+        }
+
+        if input.sig.asyncness.is_none() {
+            let msg = "the async keyword is missing from the function declaration";
+            return syn::Error::new_spanned(&input.sig.fn_token, msg)
+                .to_compile_error()
+                .into();
+        } else if !input.sig.inputs.is_empty() {
+            let msg = "the test function cannot accept arguments";
+            return syn::Error::new_spanned(&input.sig.inputs, msg)
+                .to_compile_error()
+                .into();
+        }
+
+        let mut runtime = Runtime::Auto;
+
+        for arg in args {
+            if let syn::NestedMeta::Meta(syn::Meta::Path(path)) = arg {
+                let ident = path.get_ident();
+                if ident.is_none() {
+                    let msg = "Must have specified ident";
+                    return syn::Error::new_spanned(path, msg).to_compile_error().into();
+                }
+                match ident.unwrap().to_string().to_lowercase().as_str() {
+                    "threaded_scheduler" => runtime = Runtime::Threaded,
+                    "basic_scheduler" => runtime = Runtime::Basic,
+                    name => {
+                        let msg = format!("Unknown attribute {} is specified; expected `basic_scheduler` or `threaded_scheduler`", name);
+                        return syn::Error::new_spanned(path, msg).to_compile_error().into();
+                    }
+                }
+            }
+        }
+
+        let result = match runtime {
+            Runtime::Threaded => quote! {
+                #[::core::prelude::v1::test]
+                #(#attrs)*
+                #vis fn #name() #ret {
+                    tokio::runtime::Runtime::new().unwrap().block_on(async { #body })
+                }
+            },
+            Runtime::Basic | Runtime::Auto => quote! {
+                #[::core::prelude::v1::test]
+                #(#attrs)*
+                #vis fn #name() #ret {
+                    tokio::runtime::Builder::new()
+                        .basic_scheduler()
+                        .enable_all()
+                        .build()
+                        .unwrap()
+                        .block_on(async { #body })
+                }
+            },
+        };
+
+        result.into()
+    }
+}

tokio-macros/src/lib.rs

@@ -1,11 +1,12 @@
-#![doc(html_root_url = "https://docs.rs/tokio-macros/0.2.1")]
+#![doc(html_root_url = "https://docs.rs/tokio-macros/0.2.5")]
+#![allow(clippy::needless_doctest_main)]
 #![warn(
     missing_debug_implementations,
     missing_docs,
     rust_2018_idioms,
     unreachable_pub
 )]
-#![deny(intra_doc_link_resolution_failure)]
+#![cfg_attr(docsrs, deny(broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
@@ -13,23 +14,28 @@
 
 //! Macros for use with Tokio
 
+// This `extern` is required for older `rustc` versions but newer `rustc`
+// versions warn about the unused `extern crate`.
+#[allow(unused_extern_crates)]
 extern crate proc_macro;
 
+mod entry;
+mod select;
+
 use proc_macro::TokenStream;
-use quote::quote;
 
-enum Runtime {
-    Basic,
-    Threaded,
-    Auto,
-}
-
-/// Marks async function to be executed by selected runtime.
+/// Marks async function to be executed by selected runtime. This macro helps set up a `Runtime`
+/// without requiring the user to use [Runtime](../tokio/runtime/struct.Runtime.html) or
+/// [Builder](../tokio/runtime/struct.builder.html) directly.
 ///
 /// ## Options:
 ///
-/// - `basic_scheduler` - All tasks are executed on the current thread.
-/// - `threaded_scheduler` - Uses the multi-threaded scheduler. Used by default.
+/// If you want to set the number of worker threads used for asynchronous code, use the
+/// `core_threads` option.
+///
+/// - `core_threads=n` - Sets core threads to `n` (requires `rt-threaded` feature).
+/// - `max_threads=n` - Sets max threads to `n` (requires `rt-core` or `rt-threaded` feature).
+/// - `basic_scheduler` - Use the basic schduler (requires `rt-core`).
 ///
 /// ## Function arguments:
 ///
@@ -46,6 +52,121 @@ enum Runtime {
 /// }
 /// ```
 ///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new()
+///         .threaded_scheduler()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// ### Using basic scheduler
+///
+/// The basic scheduler is single-threaded.
+///
+/// ```rust
+/// #[tokio::main(basic_scheduler)]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new()
+///         .basic_scheduler()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// ### Set number of core threads
+///
+/// ```rust
+/// #[tokio::main(core_threads = 2)]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new()
+///         .threaded_scheduler()
+///         .core_threads(2)
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
+#[proc_macro_attribute]
+#[cfg(not(test))] // Work around for rust-lang/rust#62127
+pub fn main_threaded(args: TokenStream, item: TokenStream) -> TokenStream {
+    entry::main(args, item, true)
+}
+
+/// Marks async function to be executed by selected runtime. This macro helps set up a `Runtime`
+/// without requiring the user to use [Runtime](../tokio/runtime/struct.Runtime.html) or
+/// [Builder](../tokio/runtime/struct.builder.html) directly.
+///
+/// ## Options:
+///
+/// - `basic_scheduler` - All tasks are executed on the current thread.
+/// - `threaded_scheduler` - Uses the multi-threaded scheduler. Used by default (requires `rt-threaded` feature).
+///
+/// ## Function arguments:
+///
+/// Arguments are allowed for any functions aside from `main` which is special
+///
+/// ## Usage
+///
+/// ### Using default
+///
+/// ```rust
+/// #[tokio::main]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Runtime::new()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
 /// ### Select runtime
 ///
 /// ```rust
@@ -54,80 +175,131 @@ enum Runtime {
 ///     println!("Hello world");
 /// }
 /// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new()
+///         .basic_scheduler()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
 #[proc_macro_attribute]
 #[cfg(not(test))] // Work around for rust-lang/rust#62127
 pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
-    let input = syn::parse_macro_input!(item as syn::ItemFn);
-    let args = syn::parse_macro_input!(args as syn::AttributeArgs);
-
-    let ret = &input.sig.output;
-    let name = &input.sig.ident;
-    let inputs = &input.sig.inputs;
-    let body = &input.block;
-    let attrs = &input.attrs;
-    let vis = input.vis;
-
-    if input.sig.asyncness.is_none() {
-        let msg = "the async keyword is missing from the function declaration";
-        return syn::Error::new_spanned(input.sig.fn_token, msg)
-            .to_compile_error()
-            .into();
-    } else if name == "main" && !inputs.is_empty() {
-        let msg = "the main function cannot accept arguments";
-        return syn::Error::new_spanned(&input.sig.inputs, msg)
-            .to_compile_error()
-            .into();
-    }
-
-    let mut runtime = Runtime::Auto;
-
-    for arg in args {
-        if let syn::NestedMeta::Meta(syn::Meta::Path(path)) = arg {
-            let ident = path.get_ident();
-            if ident.is_none() {
-                let msg = "Must have specified ident";
-                return syn::Error::new_spanned(path, msg).to_compile_error().into();
-            }
-            match ident.unwrap().to_string().to_lowercase().as_str() {
-                "threaded_scheduler" => runtime = Runtime::Threaded,
-                "basic_scheduler" => runtime = Runtime::Basic,
-                name => {
-                    let msg = format!("Unknown attribute {} is specified; expected `basic_scheduler` or `threaded_scheduler`", name);
-                    return syn::Error::new_spanned(path, msg).to_compile_error().into();
-                }
-            }
-        }
-    }
-
-    let result = match runtime {
-        Runtime::Threaded | Runtime::Auto => quote! {
-            #(#attrs)*
-            #vis fn #name(#inputs) #ret {
-                tokio::runtime::Runtime::new().unwrap().block_on(async { #body })
-            }
-        },
-        Runtime::Basic => quote! {
-            #(#attrs)*
-            #vis fn #name(#inputs) #ret {
-                tokio::runtime::Builder::new()
-                    .basic_scheduler()
-                    .enable_all()
-                    .build()
-                    .unwrap()
-                    .block_on(async { #body })
-            }
-        },
-    };
-
-    result.into()
+    entry::old::main(args, item)
 }
 
-/// Marks async function to be executed by runtime, suitable to test enviornment
+/// Marks async function to be executed by selected runtime. This macro helps set up a `Runtime`
+/// without requiring the user to use [Runtime](../tokio/runtime/struct.Runtime.html) or
+/// [Builder](../tokio/runtime/struct.builder.html) directly.
+///
+/// ## Options:
+///
+/// - `max_threads=n` - Sets max threads to `n`.
+///
+/// ## Function arguments:
+///
+/// Arguments are allowed for any functions aside from `main` which is special
+///
+/// ## Usage
+///
+/// ### Using default
+///
+/// ```rust
+/// #[tokio::main]
+/// async fn main() {
+///     println!("Hello world");
+/// }
+/// ```
+///
+/// Equivalent code not using `#[tokio::main]`
+///
+/// ```rust
+/// fn main() {
+///     tokio::runtime::Builder::new()
+///         .basic_scheduler()
+///         .enable_all()
+///         .build()
+///         .unwrap()
+///         .block_on(async {
+///             println!("Hello world");
+///         })
+/// }
+/// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
+#[proc_macro_attribute]
+#[cfg(not(test))] // Work around for rust-lang/rust#62127
+pub fn main_basic(args: TokenStream, item: TokenStream) -> TokenStream {
+    entry::main(args, item, false)
+}
+
+/// Marks async function to be executed by runtime, suitable to test environment
+///
+/// ## Options:
+///
+/// - `core_threads=n` - Sets core threads to `n` (requires `rt-threaded` feature).
+/// - `max_threads=n` - Sets max threads to `n` (requires `rt-core` or `rt-threaded` feature).
+///
+/// ## Usage
+///
+/// ### Select runtime
+///
+/// ```no_run
+/// #[tokio::test(core_threads = 1)]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// ### Using default
+///
+/// ```no_run
+/// #[tokio::test]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
+#[proc_macro_attribute]
+pub fn test_threaded(args: TokenStream, item: TokenStream) -> TokenStream {
+    entry::test(args, item, true)
+}
+
+/// Marks async function to be executed by runtime, suitable to test environment
 ///
 /// ## Options:
 ///
 /// - `basic_scheduler` - All tasks are executed on the current thread. Used by default.
-/// - `threaded_scheduler` - Use multi-threaded scheduler.
+/// - `threaded_scheduler` - Use multi-threaded scheduler (requires `rt-threaded` feature).
 ///
 /// ## Usage
 ///
@@ -148,79 +320,50 @@ pub fn main(args: TokenStream, item: TokenStream) -> TokenStream {
 ///     assert!(true);
 /// }
 /// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
 #[proc_macro_attribute]
 pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
-    let input = syn::parse_macro_input!(item as syn::ItemFn);
-    let args = syn::parse_macro_input!(args as syn::AttributeArgs);
-
-    let ret = &input.sig.output;
-    let name = &input.sig.ident;
-    let body = &input.block;
-    let attrs = &input.attrs;
-    let vis = input.vis;
-
-    for attr in attrs {
-        if attr.path.is_ident("test") {
-            let msg = "second test attribute is supplied";
-            return syn::Error::new_spanned(&attr, msg)
-                .to_compile_error()
-                .into();
-        }
-    }
-
-    if input.sig.asyncness.is_none() {
-        let msg = "the async keyword is missing from the function declaration";
-        return syn::Error::new_spanned(&input.sig.fn_token, msg)
-            .to_compile_error()
-            .into();
-    } else if !input.sig.inputs.is_empty() {
-        let msg = "the test function cannot accept arguments";
-        return syn::Error::new_spanned(&input.sig.inputs, msg)
-            .to_compile_error()
-            .into();
-    }
-
-    let mut runtime = Runtime::Auto;
-
-    for arg in args {
-        if let syn::NestedMeta::Meta(syn::Meta::Path(path)) = arg {
-            let ident = path.get_ident();
-            if ident.is_none() {
-                let msg = "Must have specified ident";
-                return syn::Error::new_spanned(path, msg).to_compile_error().into();
-            }
-            match ident.unwrap().to_string().to_lowercase().as_str() {
-                "threaded_scheduler" => runtime = Runtime::Threaded,
-                "basic_scheduler" => runtime = Runtime::Basic,
-                name => {
-                    let msg = format!("Unknown attribute {} is specified; expected `basic_scheduler` or `threaded_scheduler`", name);
-                    return syn::Error::new_spanned(path, msg).to_compile_error().into();
-                }
-            }
-        }
-    }
-
-    let result = match runtime {
-        Runtime::Threaded => quote! {
-            #[test]
-            #(#attrs)*
-            #vis fn #name() #ret {
-                tokio::runtime::Runtime::new().unwrap().block_on(async { #body })
-            }
-        },
-        Runtime::Basic | Runtime::Auto => quote! {
-            #[test]
-            #(#attrs)*
-            #vis fn #name() #ret {
-                tokio::runtime::Builder::new()
-                    .basic_scheduler()
-                    .enable_all()
-                    .build()
-                    .unwrap()
-                    .block_on(async { #body })
-            }
-        },
-    };
-
-    result.into()
+    entry::old::test(args, item)
+}
+
+/// Marks async function to be executed by runtime, suitable to test environment
+///
+/// ## Options:
+///
+/// - `max_threads=n` - Sets max threads to `n`.
+///
+/// ## Usage
+///
+/// ```no_run
+/// #[tokio::test]
+/// async fn my_test() {
+///     assert!(true);
+/// }
+/// ```
+///
+/// ### NOTE:
+///
+/// If you rename the tokio crate in your dependencies this macro
+/// will not work. If you must rename the 0.2 version of tokio because
+/// you're also using the 0.1 version of tokio, you _must_ make the
+/// tokio 0.2 crate available as `tokio` in the module where this
+/// macro is expanded.
+#[proc_macro_attribute]
+pub fn test_basic(args: TokenStream, item: TokenStream) -> TokenStream {
+    entry::test(args, item, false)
+}
+
+/// Implementation detail of the `select!` macro. This macro is **not** intended
+/// to be used as part of the public API and is permitted to change.
+#[proc_macro]
+#[doc(hidden)]
+pub fn select_priv_declare_output_enum(input: TokenStream) -> TokenStream {
+    select::declare_output_enum(input)
 }

tokio-macros/src/select.rs

@@ -0,0 +1,43 @@
+use proc_macro::{TokenStream, TokenTree};
+use proc_macro2::Span;
+use quote::quote;
+use syn::Ident;
+
+pub(crate) fn declare_output_enum(input: TokenStream) -> TokenStream {
+    // passed in is: `(_ _ _)` with one `_` per branch
+    let branches = match input.into_iter().next() {
+        Some(TokenTree::Group(group)) => group.stream().into_iter().count(),
+        _ => panic!("unexpected macro input"),
+    };
+
+    let variants = (0..branches)
+        .map(|num| Ident::new(&format!("_{}", num), Span::call_site()))
+        .collect::<Vec<_>>();
+
+    // Use a bitfield to track which futures completed
+    let mask = Ident::new(
+        if branches <= 8 {
+            "u8"
+        } else if branches <= 16 {
+            "u16"
+        } else if branches <= 32 {
+            "u32"
+        } else if branches <= 64 {
+            "u64"
+        } else {
+            panic!("up to 64 branches supported");
+        },
+        Span::call_site(),
+    );
+
+    TokenStream::from(quote! {
+        pub(super) enum Out<#( #variants ),*> {
+            #( #variants(#variants), )*
+            // Include a `Disabled` variant signifying that all select branches
+            // failed to resolve.
+            Disabled,
+        }
+
+        pub(super) type Mask = #mask;
+    })
+}

tokio-test/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.2.1 (April 17, 2020)
+
+- Add `Future` and `Stream` implementations for `task::Spawn<T>`.
+
 # 0.2.0 (November 25, 2019)
 
 - Initial release

tokio-test/Cargo.toml

@@ -7,13 +7,13 @@ name = "tokio-test"
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "v0.2.x" git tag.
-version = "0.2.0"
+version = "0.2.1"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-test/0.2.0/tokio_test"
+documentation = "https://docs.rs/tokio-test/0.2.1/tokio_test"
 description = """
 Testing utilities for Tokio- and futures-based code
 """

tokio-test/src/io.rs

@@ -12,7 +12,7 @@
 //!
 //! # Usage
 //!
-//! Attempting to write data that the mock isn't expected will result in a
+//! Attempting to write data that the mock isn't expecting will result in a
 //! panic.
 //!
 //! [`AsyncRead`]: tokio::io::AsyncRead
@@ -27,6 +27,7 @@ use futures_core::ready;
 use std::collections::VecDeque;
 use std::future::Future;
 use std::pin::Pin;
+use std::sync::Arc;
 use std::task::{self, Poll, Waker};
 use std::{cmp, io};
 
@@ -57,6 +58,10 @@ enum Action {
     Read(Vec<u8>),
     Write(Vec<u8>),
     Wait(Duration),
+    // Wrapped in Arc so that Builder can be cloned and Send.
+    // Mock is not cloned as does not need to check Rc for ref counts.
+    ReadError(Option<Arc<io::Error>>),
+    WriteError(Option<Arc<io::Error>>),
 }
 
 #[derive(Debug)]
@@ -83,6 +88,16 @@ impl Builder {
         self
     }
 
+    /// Sequence a `read` operation that produces an error.
+    ///
+    /// The next operation in the mock's script will be to expect a `read` call
+    /// and return `error`.
+    pub fn read_error(&mut self, error: io::Error) -> &mut Self {
+        let error = Some(error.into());
+        self.actions.push_back(Action::ReadError(error));
+        self
+    }
+
     /// Sequence a `write` operation.
     ///
     /// The next operation in the mock's script will be to expect a `write`
@@ -92,6 +107,16 @@ impl Builder {
         self
     }
 
+    /// Sequence a `write` operation that produces an error.
+    ///
+    /// The next operation in the mock's script will be to expect a `write`
+    /// call that provides `error`.
+    pub fn write_error(&mut self, error: io::Error) -> &mut Self {
+        let error = Some(error.into());
+        self.actions.push_back(Action::WriteError(error));
+        self
+    }
+
     /// Sequence a wait.
     ///
     /// The next operation in the mock's script will be to wait without doing so
@@ -128,6 +153,16 @@ impl Handle {
         self
     }
 
+    /// Sequence a `read` operation error.
+    ///
+    /// The next operation in the mock's script will be to expect a `read` call
+    /// and return `error`.
+    pub fn read_error(&mut self, error: io::Error) -> &mut Self {
+        let error = Some(error.into());
+        self.tx.send(Action::ReadError(error)).unwrap();
+        self
+    }
+
     /// Sequence a `write` operation.
     ///
     /// The next operation in the mock's script will be to expect a `write`
@@ -136,6 +171,16 @@ impl Handle {
         self.tx.send(Action::Write(buf.into())).unwrap();
         self
     }
+
+    /// Sequence a `write` operation error.
+    ///
+    /// The next operation in the mock's script will be to expect a `write`
+    /// call error.
+    pub fn write_error(&mut self, error: io::Error) -> &mut Self {
+        let error = Some(error.into());
+        self.tx.send(Action::WriteError(error)).unwrap();
+        self
+    }
 }
 
 impl Inner {
@@ -174,6 +219,12 @@ impl Inner {
                 // Return the number of bytes read
                 Ok(n)
             }
+            Some(&mut Action::ReadError(ref mut err)) => {
+                // As the
+                let err = err.take().expect("Should have been removed from actions.");
+                let err = Arc::try_unwrap(err).expect("There are no other references.");
+                Err(err)
+            }
             Some(_) => {
                 // Either waiting or expecting a write
                 Err(io::ErrorKind::WouldBlock.into())
@@ -193,6 +244,12 @@ impl Inner {
             return Err(io::ErrorKind::WouldBlock.into());
         }
 
+        if let Some(&mut Action::WriteError(ref mut err)) = self.action() {
+            let err = err.take().expect("Should have been removed from actions.");
+            let err = Arc::try_unwrap(err).expect("There are no other references.");
+            return Err(err);
+        }
+
         for i in 0..self.actions.len() {
             match self.actions[i] {
                 Action::Write(ref mut expect) => {
@@ -210,7 +267,7 @@ impl Inner {
                         return Ok(ret);
                     }
                 }
-                Action::Wait(..) => {
+                Action::Wait(..) | Action::WriteError(..) => {
                     break;
                 }
                 _ => {}
@@ -258,6 +315,11 @@ impl Inner {
                         break;
                     }
                 }
+                Action::ReadError(ref mut error) | Action::WriteError(ref mut error) => {
+                    if error.is_some() {
+                        break;
+                    }
+                }
             }
 
             let _action = self.actions.pop_front();
@@ -272,7 +334,7 @@ impl Inner {
 impl Mock {
     fn maybe_wakeup_reader(&mut self) {
         match self.inner.action() {
-            Some(&mut Action::Read(_)) | None => {
+            Some(&mut Action::Read(_)) | Some(&mut Action::ReadError(_)) | None => {
                 if let Some(waker) = self.inner.read_wait.take() {
                     waker.wake();
                 }

tokio-test/src/lib.rs

@@ -1,11 +1,11 @@
-#![doc(html_root_url = "https://docs.rs/tokio-test/0.2.0")]
+#![doc(html_root_url = "https://docs.rs/tokio-test/0.2.1")]
 #![warn(
     missing_debug_implementations,
     missing_docs,
     rust_2018_idioms,
     unreachable_pub
 )]
-#![deny(intra_doc_link_resolution_failure)]
+#![cfg_attr(docsrs, deny(broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))

tokio-test/src/macros.rs

@@ -1,6 +1,6 @@
 //! A collection of useful macros for testing futures and tokio based code
 
-/// Assert a `Poll` is ready, returning the value.
+/// Asserts a `Poll` is ready, returning the value.
 ///
 /// This will invoke `panic!` if the provided `Poll` does not evaluate to `Poll::Ready` at
 /// runtime.
@@ -39,7 +39,7 @@ macro_rules! assert_ready {
     }};
 }
 
-/// Assert a `Poll<Result<...>>` is ready and `Ok`, returning the value.
+/// Asserts a `Poll<Result<...>>` is ready and `Ok`, returning the value.
 ///
 /// This will invoke `panic!` if the provided `Poll` does not evaluate to `Poll::Ready(Ok(..))` at
 /// runtime.
@@ -72,7 +72,7 @@ macro_rules! assert_ready_ok {
     }};
 }
 
-/// Assert a `Poll<Result<...>>` is ready and `Err`, returning the error.
+/// Asserts a `Poll<Result<...>>` is ready and `Err`, returning the error.
 ///
 /// This will invoke `panic!` if the provided `Poll` does not evaluate to `Poll::Ready(Err(..))` at
 /// runtime.
@@ -105,7 +105,7 @@ macro_rules! assert_ready_err {
     }};
 }
 
-/// Assert a `Poll` is pending.
+/// Asserts a `Poll` is pending.
 ///
 /// This will invoke `panic!` if the provided `Poll` does not evaluate to `Poll::Pending` at
 /// runtime.
@@ -144,7 +144,7 @@ macro_rules! assert_pending {
     }};
 }
 
-/// Assert if a poll is ready and check for equality on the value
+/// Asserts if a poll is ready and check for equality on the value
 ///
 /// This will invoke `panic!` if the provided `Poll` does not evaluate to `Poll::Ready` at
 /// runtime and the value produced does not partially equal the expected value.

tokio-test/src/task.rs

@@ -45,22 +45,12 @@ const WAKE: usize = 1;
 const SLEEP: usize = 2;
 
 impl<T> Spawn<T> {
-    /// Consume `self` returning the inner value
-    pub fn into_inner(mut self) -> T
+    /// Consumes `self` returning the inner value
+    pub fn into_inner(self) -> T
     where
         T: Unpin,
     {
-        drop(self.task);
-
-        // Pin::into_inner is unstable, so we work around it
-        //
-        // Safety: `T` is bound by `Unpin`.
-        unsafe {
-            let ptr = Pin::get_mut(self.future.as_mut()) as *mut T;
-            let future = Box::from_raw(ptr);
-            mem::forget(self.future);
-            *future
-        }
+        *Pin::into_inner(self.future)
     }
 
     /// Returns `true` if the inner future has received a wake notification
@@ -101,7 +91,7 @@ impl<T: Unpin> ops::DerefMut for Spawn<T> {
 }
 
 impl<T: Future> Spawn<T> {
-    /// Poll a future
+    /// Polls a future
     pub fn poll(&mut self) -> Poll<T::Output> {
         let fut = self.future.as_mut();
         self.task.enter(|cx| fut.poll(cx))
@@ -109,22 +99,38 @@ impl<T: Future> Spawn<T> {
 }
 
 impl<T: Stream> Spawn<T> {
-    /// Poll a stream
+    /// Polls a stream
     pub fn poll_next(&mut self) -> Poll<Option<T::Item>> {
         let stream = self.future.as_mut();
         self.task.enter(|cx| stream.poll_next(cx))
     }
 }
 
+impl<T: Future> Future for Spawn<T> {
+    type Output = T::Output;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        self.future.as_mut().poll(cx)
+    }
+}
+
+impl<T: Stream> Stream for Spawn<T> {
+    type Item = T::Item;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.future.as_mut().poll_next(cx)
+    }
+}
+
 impl MockTask {
-    /// Create a new mock task
+    /// Creates new mock task
     fn new() -> Self {
         MockTask {
             waker: Arc::new(ThreadWaker::new()),
         }
     }
 
-    /// Run a closure from the context of the task.
+    /// Runs a closure from the context of the task.
     ///
     /// Any wake notifications resulting from the execution of the closure are
     /// tracked.
@@ -190,8 +196,7 @@ impl ThreadWaker {
     }
 
     fn wake(&self) {
-        // First, try transitioning from IDLE -> NOTIFY, this does not require a
-        // lock.
+        // First, try transitioning from IDLE -> NOTIFY, this does not require a lock.
         let mut state = self.state.lock().unwrap();
         let prev = *state;
 

tokio-test/tests/io.rs

@@ -1,5 +1,6 @@
 #![warn(rust_2018_idioms)]
 
+use std::io;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio_test::io::Builder;
 
@@ -16,6 +17,31 @@ async fn read() {
     assert_eq!(&buf[..n], b"world!");
 }
 
+#[tokio::test]
+async fn read_error() {
+    let error = io::Error::new(io::ErrorKind::Other, "cruel");
+    let mut mock = Builder::new()
+        .read(b"hello ")
+        .read_error(error)
+        .read(b"world!")
+        .build();
+    let mut buf = [0; 256];
+
+    let n = mock.read(&mut buf).await.expect("read 1");
+    assert_eq!(&buf[..n], b"hello ");
+
+    match mock.read(&mut buf).await {
+        Err(error) => {
+            assert_eq!(error.kind(), io::ErrorKind::Other);
+            assert_eq!("cruel", format!("{}", error));
+        }
+        Ok(_) => panic!("error not received"),
+    }
+
+    let n = mock.read(&mut buf).await.expect("read 1");
+    assert_eq!(&buf[..n], b"world!");
+}
+
 #[tokio::test]
 async fn write() {
     let mut mock = Builder::new().write(b"hello ").write(b"world!").build();
@@ -23,3 +49,24 @@ async fn write() {
     mock.write_all(b"hello ").await.expect("write 1");
     mock.write_all(b"world!").await.expect("write 2");
 }
+
+#[tokio::test]
+async fn write_error() {
+    let error = io::Error::new(io::ErrorKind::Other, "cruel");
+    let mut mock = Builder::new()
+        .write(b"hello ")
+        .write_error(error)
+        .write(b"world!")
+        .build();
+    mock.write_all(b"hello ").await.expect("write 1");
+
+    match mock.write_all(b"whoa").await {
+        Err(error) => {
+            assert_eq!(error.kind(), io::ErrorKind::Other);
+            assert_eq!("cruel", format!("{}", error));
+        }
+        Ok(_) => panic!("error not received"),
+    }
+
+    mock.write_all(b"world!").await.expect("write 2");
+}

tokio-tls/CHANGELOG.md

@@ -1,36 +0,0 @@
-# 0.3.0 (November 26, 2019)
-
-- Updates for tokio 0.2 release
-
-# 0.3.0-alpha.6 (September 30, 2019)
-
-- Move to `futures-*-preview 0.3.0-alpha.19`
-- Move to `pin-project 0.4`
-
-# 0.3.0-alpha.5 (September 19, 2019)
-
-### Added
-- `TlsStream::get_ref` and `TlsStream::get_mut` (#1537).
-
-# 0.3.0-alpha.4 (August 30, 2019)
-
-### Changed
-- Track `tokio` 0.2.0-alpha.4
-
-# 0.3.0-alpha.2 (August 17, 2019)
-
-### Changed
-- Update `futures` dependency to 0.3.0-alpha.18.
-
-# 0.3.0-alpha.1 (August 8, 2019)
-
-### Changed
-- Switch to `async`, `await`, and `std::future`.
-
-# 0.2.1 (January 6, 2019)
-
-* Implement `Clone` for `TlsConnector` and `TlsAcceptor` (#777)
-
-# 0.2.0 (August 8, 2018)
-
-* Initial release with `tokio` support.

tokio-tls/Cargo.toml

@@ -1,62 +0,0 @@
-[package]
-name = "tokio-tls"
-# When releasing to crates.io:
-# - Remove path dependencies
-# - Update html_root_url.
-# - Update doc url
-#   - Cargo.toml
-#   - README.md
-# - Update CHANGELOG.md.
-# - Create "v0.3.x" git tag.
-version = "0.3.0"
-edition = "2018"
-authors = ["Tokio Contributors <[email protected]>"]
-license = "MIT"
-repository = "https://github.com/tokio-rs/tokio"
-homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-tls/0.3.0-alpha.6/tokio_tls/"
-description = """
-An implementation of TLS/SSL streams for Tokio giving an implementation of TLS
-for nonblocking I/O streams.
-"""
-categories = ["asynchronous", "network-programming"]
-
-[badges]
-travis-ci = { repository = "tokio-rs/tokio-tls" }
-
-[dependencies]
-native-tls = "0.2"
-tokio = { version = "0.2.0", path = "../tokio" }
-
-[dev-dependencies]
-tokio = { version = "0.2.0", path = "../tokio", features = ["macros", "stream", "rt-core", "io-util", "net"] }
-
-cfg-if = "0.1"
-env_logger = { version = "0.6", default-features = false }
-futures = { version = "0.3.0", features = ["async-await"] }
-
-[target.'cfg(all(not(target_os = "macos"), not(windows), not(target_os = "ios")))'.dev-dependencies]
-openssl = "0.10"
-
-[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dev-dependencies]
-security-framework = "0.2"
-
-[target.'cfg(windows)'.dev-dependencies]
-schannel = "0.1"
-
-[target.'cfg(windows)'.dev-dependencies.winapi]
-version = "0.3"
-features = [
-  "lmcons",
-  "basetsd",
-  "minwinbase",
-  "minwindef",
-  "ntdef",
-  "sysinfoapi",
-  "timezoneapi",
-  "wincrypt",
-  "winerror",
-]
-
-[package.metadata.docs.rs]
-all-features = true

tokio-tls/LICENSE

@@ -1,25 +0,0 @@
-Copyright (c) 2019 Tokio Contributors
-
-Permission is hereby granted, free of charge, to any
-person obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the
-Software without restriction, including without
-limitation the rights to use, copy, modify, merge,
-publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software
-is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice
-shall be included in all copies or substantial portions
-of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
-PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
-IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.

tokio-tls/README.md

@@ -1,14 +0,0 @@
-# tokio-tls
-
-An implementation of TLS/SSL streams for Tokio built on top of the [`native-tls`
-crate]
-
-## License
-
-This project is licensed under the [MIT license](./LICENSE).
-
-### Contribution
-
-Unless you explicitly state otherwise, any contribution intentionally submitted
-for inclusion in Tokio by you, shall be licensed as MIT, without any additional
-terms or conditions.

tokio-tls/examples_old/download-rust-lang.rs

@@ -1,40 +0,0 @@
-// #![warn(rust_2018_idioms)]
-
-use native_tls::TlsConnector;
-use std::error::Error;
-use std::net::ToSocketAddrs;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::TcpStream;
-use tokio_tls;
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
-    let addr = "www.rust-lang.org:443"
-        .to_socket_addrs()?
-        .next()
-        .ok_or("failed to resolve www.rust-lang.org")?;
-
-    let socket = TcpStream::connect(&addr).await?;
-    let cx = TlsConnector::builder().build()?;
-    let cx = tokio_tls::TlsConnector::from(cx);
-
-    let mut socket = cx.connect("www.rust-lang.org", socket).await?;
-
-    socket
-        .write_all(
-            "\
-             GET / HTTP/1.0\r\n\
-             Host: www.rust-lang.org\r\n\
-             \r\n\
-             "
-            .as_bytes(),
-        )
-        .await?;
-
-    let mut data = Vec::new();
-    socket.read_to_end(&mut data).await?;
-
-    // println!("data: {:?}", &data);
-    println!("{}", String::from_utf8_lossy(&data[..]));
-    Ok(())
-}

tokio-tls/examples_old/echo.rs

@@ -1,60 +0,0 @@
-#![warn(rust_2018_idioms)]
-
-// A tiny async TLS echo server with Tokio
-use native_tls;
-use native_tls::Identity;
-use tokio;
-use tokio::io;
-use tokio::net::TcpListener;
-use tokio::prelude::*;
-use tokio_tls;
-
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Bind the server's socket
-    let addr = "127.0.0.1:12345".parse()?;
-    let tcp = TcpListener::bind(&addr)?;
-
-    // Create the TLS acceptor.
-    let der = include_bytes!("identity.p12");
-    let cert = Identity::from_pkcs12(der, "mypass")?;
-    let tls_acceptor =
-        tokio_tls::TlsAcceptor::from(native_tls::TlsAcceptor::builder(cert).build()?);
-
-    // Iterate incoming connections
-    let server = tcp
-        .incoming()
-        .for_each(move |tcp| {
-            // Accept the TLS connection.
-            let tls_accept = tls_acceptor
-                .accept(tcp)
-                .and_then(move |tls| {
-                    // Split up the read and write halves
-                    let (reader, writer) = tls.split();
-
-                    // Copy the data back to the client
-                    let conn = io::copy(reader, writer)
-                        // print what happened
-                        .map(|(n, _, _)| println!("wrote {} bytes", n))
-                        // Handle any errors
-                        .map_err(|err| println!("IO error {:?}", err));
-
-                    // Spawn the future as a concurrent task
-                    tokio::spawn(conn);
-
-                    Ok(())
-                })
-                .map_err(|err| {
-                    println!("TLS accept error: {:?}", err);
-                });
-            tokio::spawn(tls_accept);
-
-            Ok(())
-        })
-        .map_err(|err| {
-            println!("server error {:?}", err);
-        });
-
-    // Start the runtime and spin up the server
-    tokio::run(server);
-    Ok(())
-}

tokio-tls/src/lib.rs

@@ -1,361 +0,0 @@
-#![doc(html_root_url = "https://docs.rs/tokio-tls/0.3.0")]
-#![warn(
-    missing_debug_implementations,
-    missing_docs,
-    rust_2018_idioms,
-    unreachable_pub
-)]
-#![deny(intra_doc_link_resolution_failure)]
-#![doc(test(
-    no_crate_inject,
-    attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
-))]
-
-//! Async TLS streams
-//!
-//! This library is an implementation of TLS streams using the most appropriate
-//! system library by default for negotiating the connection. That is, on
-//! Windows this library uses SChannel, on OSX it uses SecureTransport, and on
-//! other platforms it uses OpenSSL.
-//!
-//! Each TLS stream implements the `Read` and `Write` traits to interact and
-//! interoperate with the rest of the futures I/O ecosystem. Client connections
-//! initiated from this crate verify hostnames automatically and by default.
-//!
-//! This crate primarily exports this ability through two newtypes,
-//! `TlsConnector` and `TlsAcceptor`. These newtypes augment the
-//! functionality provided by the `native-tls` crate, on which this crate is
-//! built. Configuration of TLS parameters is still primarily done through the
-//! `native-tls` crate.
-
-use tokio::io::{AsyncRead, AsyncWrite};
-
-use native_tls::{Error, HandshakeError, MidHandshakeTlsStream};
-use std::fmt;
-use std::future::Future;
-use std::io::{self, Read, Write};
-use std::marker::Unpin;
-use std::mem::MaybeUninit;
-use std::pin::Pin;
-use std::ptr::null_mut;
-use std::task::{Context, Poll};
-
-#[derive(Debug)]
-struct AllowStd<S> {
-    inner: S,
-    context: *mut (),
-}
-
-/// A wrapper around an underlying raw stream which implements the TLS or SSL
-/// protocol.
-///
-/// A `TlsStream<S>` represents a handshake that has been completed successfully
-/// and both the server and the client are ready for receiving and sending
-/// data. Bytes read from a `TlsStream` are decrypted from `S` and bytes written
-/// to a `TlsStream` are encrypted when passing through to `S`.
-#[derive(Debug)]
-pub struct TlsStream<S>(native_tls::TlsStream<AllowStd<S>>);
-
-/// A wrapper around a `native_tls::TlsConnector`, providing an async `connect`
-/// method.
-#[derive(Clone)]
-pub struct TlsConnector(native_tls::TlsConnector);
-
-/// A wrapper around a `native_tls::TlsAcceptor`, providing an async `accept`
-/// method.
-#[derive(Clone)]
-pub struct TlsAcceptor(native_tls::TlsAcceptor);
-
-struct MidHandshake<S>(Option<MidHandshakeTlsStream<AllowStd<S>>>);
-
-enum StartedHandshake<S> {
-    Done(TlsStream<S>),
-    Mid(MidHandshakeTlsStream<AllowStd<S>>),
-}
-
-struct StartedHandshakeFuture<F, S>(Option<StartedHandshakeFutureInner<F, S>>);
-struct StartedHandshakeFutureInner<F, S> {
-    f: F,
-    stream: S,
-}
-
-struct Guard<'a, S>(&'a mut TlsStream<S>)
-where
-    AllowStd<S>: Read + Write;
-
-impl<S> Drop for Guard<'_, S>
-where
-    AllowStd<S>: Read + Write,
-{
-    fn drop(&mut self) {
-        (self.0).0.get_mut().context = null_mut();
-    }
-}
-
-// *mut () context is neither Send nor Sync
-unsafe impl<S: Send> Send for AllowStd<S> {}
-unsafe impl<S: Sync> Sync for AllowStd<S> {}
-
-impl<S> AllowStd<S>
-where
-    S: Unpin,
-{
-    fn with_context<F, R>(&mut self, f: F) -> R
-    where
-        F: FnOnce(&mut Context<'_>, Pin<&mut S>) -> R,
-    {
-        unsafe {
-            assert!(!self.context.is_null());
-            let waker = &mut *(self.context as *mut _);
-            f(waker, Pin::new(&mut self.inner))
-        }
-    }
-}
-
-impl<S> Read for AllowStd<S>
-where
-    S: AsyncRead + Unpin,
-{
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        match self.with_context(|ctx, stream| stream.poll_read(ctx, buf)) {
-            Poll::Ready(r) => r,
-            Poll::Pending => Err(io::Error::from(io::ErrorKind::WouldBlock)),
-        }
-    }
-}
-
-impl<S> Write for AllowStd<S>
-where
-    S: AsyncWrite + Unpin,
-{
-    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-        match self.with_context(|ctx, stream| stream.poll_write(ctx, buf)) {
-            Poll::Ready(r) => r,
-            Poll::Pending => Err(io::Error::from(io::ErrorKind::WouldBlock)),
-        }
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        match self.with_context(|ctx, stream| stream.poll_flush(ctx)) {
-            Poll::Ready(r) => r,
-            Poll::Pending => Err(io::Error::from(io::ErrorKind::WouldBlock)),
-        }
-    }
-}
-
-fn cvt<T>(r: io::Result<T>) -> Poll<io::Result<T>> {
-    match r {
-        Ok(v) => Poll::Ready(Ok(v)),
-        Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => Poll::Pending,
-        Err(e) => Poll::Ready(Err(e)),
-    }
-}
-
-impl<S> TlsStream<S> {
-    fn with_context<F, R>(&mut self, ctx: &mut Context<'_>, f: F) -> R
-    where
-        F: FnOnce(&mut native_tls::TlsStream<AllowStd<S>>) -> R,
-        AllowStd<S>: Read + Write,
-    {
-        self.0.get_mut().context = ctx as *mut _ as *mut ();
-        let g = Guard(self);
-        f(&mut (g.0).0)
-    }
-
-    /// Returns a shared reference to the inner stream.
-    pub fn get_ref(&self) -> &S
-    where
-        S: AsyncRead + AsyncWrite + Unpin,
-    {
-        &self.0.get_ref().inner
-    }
-
-    /// Returns a mutable reference to the inner stream.
-    pub fn get_mut(&mut self) -> &mut S
-    where
-        S: AsyncRead + AsyncWrite + Unpin,
-    {
-        &mut self.0.get_mut().inner
-    }
-}
-
-impl<S> AsyncRead for TlsStream<S>
-where
-    S: AsyncRead + AsyncWrite + Unpin,
-{
-    unsafe fn prepare_uninitialized_buffer(&self, _: &mut [MaybeUninit<u8>]) -> bool {
-        // Note that this does not forward to `S` because the buffer is
-        // unconditionally filled in by OpenSSL, not the actual object `S`.
-        // We're decrypting bytes from `S` into the buffer above!
-        false
-    }
-
-    fn poll_read(
-        mut self: Pin<&mut Self>,
-        ctx: &mut Context<'_>,
-        buf: &mut [u8],
-    ) -> Poll<io::Result<usize>> {
-        self.with_context(ctx, |s| cvt(s.read(buf)))
-    }
-}
-
-impl<S> AsyncWrite for TlsStream<S>
-where
-    S: AsyncRead + AsyncWrite + Unpin,
-{
-    fn poll_write(
-        mut self: Pin<&mut Self>,
-        ctx: &mut Context<'_>,
-        buf: &[u8],
-    ) -> Poll<io::Result<usize>> {
-        self.with_context(ctx, |s| cvt(s.write(buf)))
-    }
-
-    fn poll_flush(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        self.with_context(ctx, |s| cvt(s.flush()))
-    }
-
-    fn poll_shutdown(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll<io::Result<()>> {
-        match self.with_context(ctx, |s| s.shutdown()) {
-            Ok(()) => Poll::Ready(Ok(())),
-            Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => Poll::Pending,
-            Err(e) => Poll::Ready(Err(e)),
-        }
-    }
-}
-
-async fn handshake<F, S>(f: F, stream: S) -> Result<TlsStream<S>, Error>
-where
-    F: FnOnce(
-            AllowStd<S>,
-        ) -> Result<native_tls::TlsStream<AllowStd<S>>, HandshakeError<AllowStd<S>>>
-        + Unpin,
-    S: AsyncRead + AsyncWrite + Unpin,
-{
-    let start = StartedHandshakeFuture(Some(StartedHandshakeFutureInner { f, stream }));
-
-    match start.await {
-        Err(e) => Err(e),
-        Ok(StartedHandshake::Done(s)) => Ok(s),
-        Ok(StartedHandshake::Mid(s)) => MidHandshake(Some(s)).await,
-    }
-}
-
-impl<F, S> Future for StartedHandshakeFuture<F, S>
-where
-    F: FnOnce(
-            AllowStd<S>,
-        ) -> Result<native_tls::TlsStream<AllowStd<S>>, HandshakeError<AllowStd<S>>>
-        + Unpin,
-    S: Unpin,
-    AllowStd<S>: Read + Write,
-{
-    type Output = Result<StartedHandshake<S>, Error>;
-
-    fn poll(
-        mut self: Pin<&mut Self>,
-        ctx: &mut Context<'_>,
-    ) -> Poll<Result<StartedHandshake<S>, Error>> {
-        let inner = self.0.take().expect("future polled after completion");
-        let stream = AllowStd {
-            inner: inner.stream,
-            context: ctx as *mut _ as *mut (),
-        };
-
-        match (inner.f)(stream) {
-            Ok(mut s) => {
-                s.get_mut().context = null_mut();
-                Poll::Ready(Ok(StartedHandshake::Done(TlsStream(s))))
-            }
-            Err(HandshakeError::WouldBlock(mut s)) => {
-                s.get_mut().context = null_mut();
-                Poll::Ready(Ok(StartedHandshake::Mid(s)))
-            }
-            Err(HandshakeError::Failure(e)) => Poll::Ready(Err(e)),
-        }
-    }
-}
-
-impl TlsConnector {
-    /// Connects the provided stream with this connector, assuming the provided
-    /// domain.
-    ///
-    /// This function will internally call `TlsConnector::connect` to connect
-    /// the stream and returns a future representing the resolution of the
-    /// connection operation. The returned future will resolve to either
-    /// `TlsStream<S>` or `Error` depending if it's successful or not.
-    ///
-    /// This is typically used for clients who have already established, for
-    /// example, a TCP connection to a remote server. That stream is then
-    /// provided here to perform the client half of a connection to a
-    /// TLS-powered server.
-    pub async fn connect<S>(&self, domain: &str, stream: S) -> Result<TlsStream<S>, Error>
-    where
-        S: AsyncRead + AsyncWrite + Unpin,
-    {
-        handshake(move |s| self.0.connect(domain, s), stream).await
-    }
-}
-
-impl fmt::Debug for TlsConnector {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("TlsConnector").finish()
-    }
-}
-
-impl From<native_tls::TlsConnector> for TlsConnector {
-    fn from(inner: native_tls::TlsConnector) -> TlsConnector {
-        TlsConnector(inner)
-    }
-}
-
-impl TlsAcceptor {
-    /// Accepts a new client connection with the provided stream.
-    ///
-    /// This function will internally call `TlsAcceptor::accept` to connect
-    /// the stream and returns a future representing the resolution of the
-    /// connection operation. The returned future will resolve to either
-    /// `TlsStream<S>` or `Error` depending if it's successful or not.
-    ///
-    /// This is typically used after a new socket has been accepted from a
-    /// `TcpListener`. That socket is then passed to this function to perform
-    /// the server half of accepting a client connection.
-    pub async fn accept<S>(&self, stream: S) -> Result<TlsStream<S>, Error>
-    where
-        S: AsyncRead + AsyncWrite + Unpin,
-    {
-        handshake(move |s| self.0.accept(s), stream).await
-    }
-}
-
-impl fmt::Debug for TlsAcceptor {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("TlsAcceptor").finish()
-    }
-}
-
-impl From<native_tls::TlsAcceptor> for TlsAcceptor {
-    fn from(inner: native_tls::TlsAcceptor) -> TlsAcceptor {
-        TlsAcceptor(inner)
-    }
-}
-
-impl<S: AsyncRead + AsyncWrite + Unpin> Future for MidHandshake<S> {
-    type Output = Result<TlsStream<S>, Error>;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let mut_self = self.get_mut();
-        let mut s = mut_self.0.take().expect("future polled after completion");
-
-        s.get_mut().context = cx as *mut _ as *mut ();
-        match s.handshake() {
-            Ok(stream) => Poll::Ready(Ok(TlsStream(stream))),
-            Err(HandshakeError::Failure(e)) => Poll::Ready(Err(e)),
-            Err(HandshakeError::WouldBlock(mut s)) => {
-                s.get_mut().context = null_mut();
-                mut_self.0 = Some(s);
-                Poll::Pending
-            }
-        }
-    }
-}

tokio-tls/tests/bad.rs

@@ -1,124 +0,0 @@
-#![warn(rust_2018_idioms)]
-
-use cfg_if::cfg_if;
-use env_logger;
-use native_tls::TlsConnector;
-use std::io::{self, Error};
-use std::net::ToSocketAddrs;
-use tokio::net::TcpStream;
-use tokio_tls;
-
-macro_rules! t {
-    ($e:expr) => {
-        match $e {
-            Ok(e) => e,
-            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
-        }
-    };
-}
-
-cfg_if! {
-    if #[cfg(feature = "force-rustls")] {
-        fn verify_failed(err: &Error, s:  &str) {
-            let err = err.to_string();
-            assert!(err.contains(s), "bad error: {}", err);
-        }
-
-        fn assert_expired_error(err: &Error) {
-            verify_failed(err, "CertExpired");
-        }
-
-        fn assert_wrong_host(err: &Error) {
-            verify_failed(err, "CertNotValidForName");
-        }
-
-        fn assert_self_signed(err: &Error) {
-            verify_failed(err, "UnknownIssuer");
-        }
-
-        fn assert_untrusted_root(err: &Error) {
-            verify_failed(err, "UnknownIssuer");
-        }
-    } else if #[cfg(any(feature = "force-openssl",
-                        all(not(target_os = "macos"),
-                            not(target_os = "windows"),
-                            not(target_os = "ios"))))] {
-        fn verify_failed(err: &Error) {
-            assert!(format!("{}", err).contains("certificate verify failed"))
-        }
-
-        use verify_failed as assert_expired_error;
-        use verify_failed as assert_wrong_host;
-        use verify_failed as assert_self_signed;
-        use verify_failed as assert_untrusted_root;
-    } else if #[cfg(any(target_os = "macos", target_os = "ios"))] {
-
-        fn assert_invalid_cert_chain(err: &Error) {
-            assert!(format!("{}", err).contains("was not trusted."))
-        }
-
-        use crate::assert_invalid_cert_chain as assert_expired_error;
-        use crate::assert_invalid_cert_chain as assert_wrong_host;
-        use crate::assert_invalid_cert_chain as assert_self_signed;
-        use crate::assert_invalid_cert_chain as assert_untrusted_root;
-    } else {
-        fn assert_expired_error(err: &Error) {
-            let s = err.to_string();
-            assert!(s.contains("system clock"), "error = {:?}", s);
-        }
-
-        fn assert_wrong_host(err: &Error) {
-            let s = err.to_string();
-            assert!(s.contains("CN name"), "error = {:?}", s);
-        }
-
-        fn assert_self_signed(err: &Error) {
-            let s = err.to_string();
-            assert!(s.contains("root certificate which is not trusted"), "error = {:?}", s);
-        }
-
-        use assert_self_signed as assert_untrusted_root;
-    }
-}
-
-async fn get_host(host: &'static str) -> Error {
-    drop(env_logger::try_init());
-
-    let addr = format!("{}:443", host);
-    let addr = t!(addr.to_socket_addrs()).next().unwrap();
-
-    let socket = t!(TcpStream::connect(&addr).await);
-    let builder = TlsConnector::builder();
-    let cx = t!(builder.build());
-    let cx = tokio_tls::TlsConnector::from(cx);
-    let res = cx
-        .connect(host, socket)
-        .await
-        .map_err(|e| Error::new(io::ErrorKind::Other, e));
-
-    assert!(res.is_err());
-    res.err().unwrap()
-}
-
-#[tokio::test]
-async fn expired() {
-    assert_expired_error(&get_host("expired.badssl.com").await)
-}
-
-// TODO: the OSX builders on Travis apparently fail this tests spuriously?
-//       passes locally though? Seems... bad!
-#[tokio::test]
-#[cfg_attr(all(target_os = "macos", feature = "force-openssl"), ignore)]
-async fn wrong_host() {
-    assert_wrong_host(&get_host("wrong.host.badssl.com").await)
-}
-
-#[tokio::test]
-async fn self_signed() {
-    assert_self_signed(&get_host("self-signed.badssl.com").await)
-}
-
-#[tokio::test]
-async fn untrusted_root() {
-    assert_untrusted_root(&get_host("untrusted-root.badssl.com").await)
-}

tokio-tls/tests/google.rs

@@ -1,102 +0,0 @@
-#![warn(rust_2018_idioms)]
-
-use cfg_if::cfg_if;
-use env_logger;
-use native_tls;
-use native_tls::TlsConnector;
-use std::io;
-use std::net::ToSocketAddrs;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use tokio::net::TcpStream;
-use tokio_tls;
-
-macro_rules! t {
-    ($e:expr) => {
-        match $e {
-            Ok(e) => e,
-            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
-        }
-    };
-}
-
-cfg_if! {
-    if #[cfg(feature = "force-rustls")] {
-        fn assert_bad_hostname_error(err: &io::Error) {
-            let err = err.to_string();
-            assert!(err.contains("CertNotValidForName"), "bad error: {}", err);
-        }
-    } else if #[cfg(any(feature = "force-openssl",
-                        all(not(target_os = "macos"),
-                            not(target_os = "windows"),
-                            not(target_os = "ios"))))] {
-        fn assert_bad_hostname_error(err: &io::Error) {
-            let err = err.get_ref().unwrap();
-            let err = err.downcast_ref::<native_tls::Error>().unwrap();
-            assert!(format!("{}", err).contains("certificate verify failed"));
-        }
-    } else if #[cfg(any(target_os = "macos", target_os = "ios"))] {
-        fn assert_bad_hostname_error(err: &io::Error) {
-            let err = err.get_ref().unwrap();
-            let err = err.downcast_ref::<native_tls::Error>().unwrap();
-            assert!(format!("{}", err).contains("was not trusted."));
-        }
-    } else {
-        fn assert_bad_hostname_error(err: &io::Error) {
-            let err = err.get_ref().unwrap();
-            let err = err.downcast_ref::<native_tls::Error>().unwrap();
-            assert!(format!("{}", err).contains("CN name"));
-        }
-    }
-}
-
-#[tokio::test]
-async fn fetch_google() {
-    drop(env_logger::try_init());
-
-    // First up, resolve google.com
-    let addr = t!("google.com:443".to_socket_addrs()).next().unwrap();
-
-    let socket = TcpStream::connect(&addr).await.unwrap();
-
-    // Send off the request by first negotiating an SSL handshake, then writing
-    // of our request, then flushing, then finally read off the response.
-    let builder = TlsConnector::builder();
-    let connector = t!(builder.build());
-    let connector = tokio_tls::TlsConnector::from(connector);
-    let mut socket = t!(connector.connect("google.com", socket).await);
-    t!(socket.write_all(b"GET / HTTP/1.0\r\n\r\n").await);
-    let mut data = Vec::new();
-    t!(socket.read_to_end(&mut data).await);
-
-    // any response code is fine
-    assert!(data.starts_with(b"HTTP/1.0 "));
-
-    let data = String::from_utf8_lossy(&data);
-    let data = data.trim_end();
-    assert!(data.ends_with("</html>") || data.ends_with("</HTML>"));
-}
-
-fn native2io(e: native_tls::Error) -> io::Error {
-    io::Error::new(io::ErrorKind::Other, e)
-}
-
-// see comment in bad.rs for ignore reason
-#[cfg_attr(all(target_os = "macos", feature = "force-openssl"), ignore)]
-#[tokio::test]
-async fn wrong_hostname_error() {
-    drop(env_logger::try_init());
-
-    let addr = t!("google.com:443".to_socket_addrs()).next().unwrap();
-
-    let socket = t!(TcpStream::connect(&addr).await);
-    let builder = TlsConnector::builder();
-    let connector = t!(builder.build());
-    let connector = tokio_tls::TlsConnector::from(connector);
-    let res = connector
-        .connect("rust-lang.org", socket)
-        .await
-        .map_err(native2io);
-
-    assert!(res.is_err());
-    assert_bad_hostname_error(&res.err().unwrap());
-}

tokio-tls/tests/smoke.rs

@@ -1,629 +0,0 @@
-#![warn(rust_2018_idioms)]
-
-use cfg_if::cfg_if;
-use env_logger;
-use futures::join;
-use native_tls;
-use native_tls::{Identity, TlsAcceptor, TlsConnector};
-use std::io::Write;
-use std::marker::Unpin;
-use std::process::Command;
-use std::ptr;
-use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, Error, ErrorKind};
-use tokio::net::{TcpListener, TcpStream};
-use tokio::stream::StreamExt;
-use tokio_tls;
-
-macro_rules! t {
-    ($e:expr) => {
-        match $e {
-            Ok(e) => e,
-            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
-        }
-    };
-}
-
-#[allow(dead_code)]
-struct Keys {
-    cert_der: Vec<u8>,
-    pkey_der: Vec<u8>,
-    pkcs12_der: Vec<u8>,
-}
-
-#[allow(dead_code)]
-fn openssl_keys() -> &'static Keys {
-    static INIT: Once = Once::new();
-    static mut KEYS: *mut Keys = ptr::null_mut();
-
-    INIT.call_once(|| {
-        let path = t!(env::current_exe());
-        let path = path.parent().unwrap();
-        let keyfile = path.join("test.key");
-        let certfile = path.join("test.crt");
-        let config = path.join("openssl.config");
-
-        File::create(&config)
-            .unwrap()
-            .write_all(
-                b"\
-            [req]\n\
-            distinguished_name=dn\n\
-            [ dn ]\n\
-            CN=localhost\n\
-            [ ext ]\n\
-            basicConstraints=CA:FALSE,pathlen:0\n\
-            subjectAltName = @alt_names
-            extendedKeyUsage=serverAuth,clientAuth
-            [alt_names]
-            DNS.1 = localhost
-        ",
-            )
-            .unwrap();
-
-        let subj = "/C=US/ST=Denial/L=Sprintfield/O=Dis/CN=localhost";
-        let output = t!(Command::new("openssl")
-            .arg("req")
-            .arg("-nodes")
-            .arg("-x509")
-            .arg("-newkey")
-            .arg("rsa:2048")
-            .arg("-config")
-            .arg(&config)
-            .arg("-extensions")
-            .arg("ext")
-            .arg("-subj")
-            .arg(subj)
-            .arg("-keyout")
-            .arg(&keyfile)
-            .arg("-out")
-            .arg(&certfile)
-            .arg("-days")
-            .arg("1")
-            .output());
-        assert!(output.status.success());
-
-        let crtout = t!(Command::new("openssl")
-            .arg("x509")
-            .arg("-outform")
-            .arg("der")
-            .arg("-in")
-            .arg(&certfile)
-            .output());
-        assert!(crtout.status.success());
-        let keyout = t!(Command::new("openssl")
-            .arg("rsa")
-            .arg("-outform")
-            .arg("der")
-            .arg("-in")
-            .arg(&keyfile)
-            .output());
-        assert!(keyout.status.success());
-
-        let pkcs12out = t!(Command::new("openssl")
-            .arg("pkcs12")
-            .arg("-export")
-            .arg("-nodes")
-            .arg("-inkey")
-            .arg(&keyfile)
-            .arg("-in")
-            .arg(&certfile)
-            .arg("-password")
-            .arg("pass:foobar")
-            .output());
-        assert!(pkcs12out.status.success());
-
-        let keys = Box::new(Keys {
-            cert_der: crtout.stdout,
-            pkey_der: keyout.stdout,
-            pkcs12_der: pkcs12out.stdout,
-        });
-        unsafe {
-            KEYS = Box::into_raw(keys);
-        }
-    });
-    unsafe { &*KEYS }
-}
-
-cfg_if! {
-    if #[cfg(feature = "rustls")] {
-        use webpki;
-        use untrusted;
-        use std::env;
-        use std::fs::File;
-        use std::process::Command;
-        use std::sync::Once;
-
-        use untrusted::Input;
-        use webpki::trust_anchor_util;
-
-        fn server_cx() -> io::Result<ServerContext> {
-            let mut cx = ServerContext::new();
-
-            let (cert, key) = keys();
-            cx.config_mut()
-              .set_single_cert(vec![cert.to_vec()], key.to_vec());
-
-            Ok(cx)
-        }
-
-        fn configure_client(cx: &mut ClientContext) {
-            let (cert, _key) = keys();
-            let cert = Input::from(cert);
-            let anchor = trust_anchor_util::cert_der_as_trust_anchor(cert).unwrap();
-            cx.config_mut().root_store.add_trust_anchors(&[anchor]);
-        }
-
-        // Like OpenSSL we generate certificates on the fly, but for OSX we
-        // also have to put them into a specific keychain. We put both the
-        // certificates and the keychain next to our binary.
-        //
-        // Right now I don't know of a way to programmatically create a
-        // self-signed certificate, so we just fork out to the `openssl` binary.
-        fn keys() -> (&'static [u8], &'static [u8]) {
-            static INIT: Once = Once::new();
-            static mut KEYS: *mut (Vec<u8>, Vec<u8>) = ptr::null_mut();
-
-            INIT.call_once(|| {
-                let (key, cert) = openssl_keys();
-                let path = t!(env::current_exe());
-                let path = path.parent().unwrap();
-                let keyfile = path.join("test.key");
-                let certfile = path.join("test.crt");
-                let config = path.join("openssl.config");
-
-                File::create(&config).unwrap().write_all(b"\
-                    [req]\n\
-                    distinguished_name=dn\n\
-                    [ dn ]\n\
-                    CN=localhost\n\
-                    [ ext ]\n\
-                    basicConstraints=CA:FALSE,pathlen:0\n\
-                    subjectAltName = @alt_names
-                    [alt_names]
-                    DNS.1 = localhost
-                ").unwrap();
-
-                let subj = "/C=US/ST=Denial/L=Sprintfield/O=Dis/CN=localhost";
-                let output = t!(Command::new("openssl")
-                                        .arg("req")
-                                        .arg("-nodes")
-                                        .arg("-x509")
-                                        .arg("-newkey").arg("rsa:2048")
-                                        .arg("-config").arg(&config)
-                                        .arg("-extensions").arg("ext")
-                                        .arg("-subj").arg(subj)
-                                        .arg("-keyout").arg(&keyfile)
-                                        .arg("-out").arg(&certfile)
-                                        .arg("-days").arg("1")
-                                        .output());
-                assert!(output.status.success());
-
-                let crtout = t!(Command::new("openssl")
-                                        .arg("x509")
-                                        .arg("-outform").arg("der")
-                                        .arg("-in").arg(&certfile)
-                                        .output());
-                assert!(crtout.status.success());
-                let keyout = t!(Command::new("openssl")
-                                        .arg("rsa")
-                                        .arg("-outform").arg("der")
-                                        .arg("-in").arg(&keyfile)
-                                        .output());
-                assert!(keyout.status.success());
-
-                let cert = crtout.stdout;
-                let key = keyout.stdout;
-                unsafe {
-                    KEYS = Box::into_raw(Box::new((cert, key)));
-                }
-            });
-            unsafe {
-                (&(*KEYS).0, &(*KEYS).1)
-            }
-        }
-    } else if #[cfg(any(feature = "force-openssl",
-                        all(not(target_os = "macos"),
-                            not(target_os = "windows"),
-                            not(target_os = "ios"))))] {
-        use std::fs::File;
-        use std::env;
-        use std::sync::Once;
-
-        fn contexts() -> (tokio_tls::TlsAcceptor, tokio_tls::TlsConnector) {
-            let keys = openssl_keys();
-
-            let pkcs12 = t!(Identity::from_pkcs12(&keys.pkcs12_der, "foobar"));
-            let srv = TlsAcceptor::builder(pkcs12);
-
-            let cert = t!(native_tls::Certificate::from_der(&keys.cert_der));
-
-            let mut client = TlsConnector::builder();
-            t!(client.add_root_certificate(cert).build());
-
-            (t!(srv.build()).into(), t!(client.build()).into())
-        }
-    } else if #[cfg(any(target_os = "macos", target_os = "ios"))] {
-        use std::env;
-        use std::fs::File;
-        use std::sync::Once;
-
-        fn contexts() -> (tokio_tls::TlsAcceptor, tokio_tls::TlsConnector) {
-            let keys = openssl_keys();
-
-            let pkcs12 = t!(Identity::from_pkcs12(&keys.pkcs12_der, "foobar"));
-            let srv = TlsAcceptor::builder(pkcs12);
-
-            let cert = native_tls::Certificate::from_der(&keys.cert_der).unwrap();
-            let mut client = TlsConnector::builder();
-            client.add_root_certificate(cert);
-
-            (t!(srv.build()).into(), t!(client.build()).into())
-        }
-    } else {
-        use schannel;
-        use winapi;
-
-        use std::env;
-        use std::fs::File;
-        use std::io;
-        use std::mem;
-        use std::sync::Once;
-
-        use schannel::cert_context::CertContext;
-        use schannel::cert_store::{CertStore, CertAdd, Memory};
-        use winapi::shared::basetsd::*;
-        use winapi::shared::lmcons::*;
-        use winapi::shared::minwindef::*;
-        use winapi::shared::ntdef::WCHAR;
-        use winapi::um::minwinbase::*;
-        use winapi::um::sysinfoapi::*;
-        use winapi::um::timezoneapi::*;
-        use winapi::um::wincrypt::*;
-
-        const FRIENDLY_NAME: &'static str = "tokio-tls localhost testing cert";
-
-        fn contexts() -> (tokio_tls::TlsAcceptor, tokio_tls::TlsConnector) {
-            let cert = localhost_cert();
-            let mut store = t!(Memory::new()).into_store();
-            t!(store.add_cert(&cert, CertAdd::Always));
-            let pkcs12_der = t!(store.export_pkcs12("foobar"));
-            let pkcs12 = t!(Identity::from_pkcs12(&pkcs12_der, "foobar"));
-
-            let srv = TlsAcceptor::builder(pkcs12);
-            let client = TlsConnector::builder();
-            (t!(srv.build()).into(), t!(client.build()).into())
-        }
-
-        // ====================================================================
-        // Magic!
-        //
-        // Lots of magic is happening here to wrangle certificates for running
-        // these tests on Windows. For more information see the test suite
-        // in the schannel-rs crate as this is just coyping that.
-        //
-        // The general gist of this though is that the only way to add custom
-        // trusted certificates is to add it to the system store of trust. To
-        // do that we go through the whole rigamarole here to generate a new
-        // self-signed certificate and then insert that into the system store.
-        //
-        // This generates some dialogs, so we print what we're doing sometimes,
-        // and otherwise we just manage the ephemeral certificates. Because
-        // they're in the system store we always ensure that they're only valid
-        // for a small period of time (e.g. 1 day).
-
-        fn localhost_cert() -> CertContext {
-            static INIT: Once = Once::new();
-            INIT.call_once(|| {
-                for cert in local_root_store().certs() {
-                    let name = match cert.friendly_name() {
-                        Ok(name) => name,
-                        Err(_) => continue,
-                    };
-                    if name != FRIENDLY_NAME {
-                        continue
-                    }
-                    if !cert.is_time_valid().unwrap() {
-                        io::stdout().write_all(br#"
-
-The tokio-tls test suite is about to delete an old copy of one of its
-certificates from your root trust store. This certificate was only valid for one
-day and it is no longer needed. The host should be "localhost" and the
-description should mention "tokio-tls".
-
-        "#).unwrap();
-                        cert.delete().unwrap();
-                    } else {
-                        return
-                    }
-                }
-
-                install_certificate().unwrap();
-            });
-
-            for cert in local_root_store().certs() {
-                let name = match cert.friendly_name() {
-                    Ok(name) => name,
-                    Err(_) => continue,
-                };
-                if name == FRIENDLY_NAME {
-                    return cert
-                }
-            }
-
-            panic!("couldn't find a cert");
-        }
-
-        fn local_root_store() -> CertStore {
-            if env::var("CI").is_ok() {
-                CertStore::open_local_machine("Root").unwrap()
-            } else {
-                CertStore::open_current_user("Root").unwrap()
-            }
-        }
-
-        fn install_certificate() -> io::Result<CertContext> {
-            unsafe {
-                let mut provider = 0;
-                let mut hkey = 0;
-
-                let mut buffer = "tokio-tls test suite".encode_utf16()
-                                                         .chain(Some(0))
-                                                         .collect::<Vec<_>>();
-                let res = CryptAcquireContextW(&mut provider,
-                                               buffer.as_ptr(),
-                                               ptr::null_mut(),
-                                               PROV_RSA_FULL,
-                                               CRYPT_MACHINE_KEYSET);
-                if res != TRUE {
-                    // create a new key container (since it does not exist)
-                    let res = CryptAcquireContextW(&mut provider,
-                                                   buffer.as_ptr(),
-                                                   ptr::null_mut(),
-                                                   PROV_RSA_FULL,
-                                                   CRYPT_NEWKEYSET | CRYPT_MACHINE_KEYSET);
-                    if res != TRUE {
-                        return Err(Error::last_os_error())
-                    }
-                }
-
-                // create a new keypair (RSA-2048)
-                let res = CryptGenKey(provider,
-                                      AT_SIGNATURE,
-                                      0x0800<<16 | CRYPT_EXPORTABLE,
-                                      &mut hkey);
-                if res != TRUE {
-                    return Err(Error::last_os_error());
-                }
-
-                // start creating the certificate
-                let name = "CN=localhost,O=tokio-tls,OU=tokio-tls,\
-                            G=tokio_tls".encode_utf16()
-                                          .chain(Some(0))
-                                          .collect::<Vec<_>>();
-                let mut cname_buffer: [WCHAR; UNLEN as usize + 1] = mem::zeroed();
-                let mut cname_len = cname_buffer.len() as DWORD;
-                let res = CertStrToNameW(X509_ASN_ENCODING,
-                                         name.as_ptr(),
-                                         CERT_X500_NAME_STR,
-                                         ptr::null_mut(),
-                                         cname_buffer.as_mut_ptr() as *mut u8,
-                                         &mut cname_len,
-                                         ptr::null_mut());
-                if res != TRUE {
-                    return Err(Error::last_os_error());
-                }
-
-                let mut subject_issuer = CERT_NAME_BLOB {
-                    cbData: cname_len,
-                    pbData: cname_buffer.as_ptr() as *mut u8,
-                };
-                let mut key_provider = CRYPT_KEY_PROV_INFO {
-                    pwszContainerName: buffer.as_mut_ptr(),
-                    pwszProvName: ptr::null_mut(),
-                    dwProvType: PROV_RSA_FULL,
-                    dwFlags: CRYPT_MACHINE_KEYSET,
-                    cProvParam: 0,
-                    rgProvParam: ptr::null_mut(),
-                    dwKeySpec: AT_SIGNATURE,
-                };
-                let mut sig_algorithm = CRYPT_ALGORITHM_IDENTIFIER {
-                    pszObjId: szOID_RSA_SHA256RSA.as_ptr() as *mut _,
-                    Parameters: mem::zeroed(),
-                };
-                let mut expiration_date: SYSTEMTIME = mem::zeroed();
-                GetSystemTime(&mut expiration_date);
-                let mut file_time: FILETIME = mem::zeroed();
-                let res = SystemTimeToFileTime(&mut expiration_date,
-                                               &mut file_time);
-                if res != TRUE {
-                    return Err(Error::last_os_error());
-                }
-                let mut timestamp: u64 = file_time.dwLowDateTime as u64 |
-                                         (file_time.dwHighDateTime as u64) << 32;
-                // one day, timestamp unit is in 100 nanosecond intervals
-                timestamp += (1E9 as u64) / 100 * (60 * 60 * 24);
-                file_time.dwLowDateTime = timestamp as u32;
-                file_time.dwHighDateTime = (timestamp >> 32) as u32;
-                let res = FileTimeToSystemTime(&file_time,
-                                               &mut expiration_date);
-                if res != TRUE {
-                    return Err(Error::last_os_error());
-                }
-
-                // create a self signed certificate
-                let cert_context = CertCreateSelfSignCertificate(
-                        0 as ULONG_PTR,
-                        &mut subject_issuer,
-                        0,
-                        &mut key_provider,
-                        &mut sig_algorithm,
-                        ptr::null_mut(),
-                        &mut expiration_date,
-                        ptr::null_mut());
-                if cert_context.is_null() {
-                    return Err(Error::last_os_error());
-                }
-
-                // TODO: this is.. a terrible hack. Right now `schannel`
-                //       doesn't provide a public method to go from a raw
-                //       cert context pointer to the `CertContext` structure it
-                //       has, so we just fake it here with a transmute. This'll
-                //       probably break at some point, but hopefully by then
-                //       it'll have a method to do this!
-                struct MyCertContext<T>(T);
-                impl<T> Drop for MyCertContext<T> {
-                    fn drop(&mut self) {}
-                }
-
-                let cert_context = MyCertContext(cert_context);
-                let cert_context: CertContext = mem::transmute(cert_context);
-
-                cert_context.set_friendly_name(FRIENDLY_NAME)?;
-
-                // install the certificate to the machine's local store
-                io::stdout().write_all(br#"
-
-The tokio-tls test suite is about to add a certificate to your set of root
-and trusted certificates. This certificate should be for the domain "localhost"
-with the description related to "tokio-tls". This certificate is only valid
-for one day and will be automatically deleted if you re-run the tokio-tls
-test suite later.
-
-        "#).unwrap();
-                local_root_store().add_cert(&cert_context,
-                                                 CertAdd::ReplaceExisting)?;
-                Ok(cert_context)
-            }
-        }
-    }
-}
-
-const AMT: usize = 128 * 1024;
-
-async fn copy_data<W: AsyncWrite + Unpin>(mut w: W) -> Result<usize, Error> {
-    let mut data = vec![9; AMT as usize];
-    let mut amt = 0;
-    while !data.is_empty() {
-        let written = w.write(&data).await?;
-        if written <= data.len() {
-            amt += written;
-            data.resize(data.len() - written, 0);
-        } else {
-            w.write_all(&data).await?;
-            amt += data.len();
-            break;
-        }
-
-        println!("remaining: {}", data.len());
-    }
-    Ok(amt)
-}
-
-#[tokio::test]
-async fn client_to_server() {
-    drop(env_logger::try_init());
-
-    // Create a server listening on a port, then figure out what that port is
-    let mut srv = t!(TcpListener::bind("127.0.0.1:0").await);
-    let addr = t!(srv.local_addr());
-
-    let (server_cx, client_cx) = contexts();
-
-    // Create a future to accept one socket, connect the ssl stream, and then
-    // read all the data from it.
-    let server = async move {
-        let mut incoming = srv.incoming();
-        let socket = t!(incoming.next().await.unwrap());
-        let mut socket = t!(server_cx.accept(socket).await);
-        let mut data = Vec::new();
-        t!(socket.read_to_end(&mut data).await);
-        data
-    };
-
-    // Create a future to connect to our server, connect the ssl stream, and
-    // then write a bunch of data to it.
-    let client = async move {
-        let socket = t!(TcpStream::connect(&addr).await);
-        let socket = t!(client_cx.connect("localhost", socket).await);
-        copy_data(socket).await
-    };
-
-    // Finally, run everything!
-    let (data, _) = join!(server, client);
-    // assert_eq!(amt, AMT);
-    assert!(data == vec![9; AMT]);
-}
-
-#[tokio::test]
-async fn server_to_client() {
-    drop(env_logger::try_init());
-
-    // Create a server listening on a port, then figure out what that port is
-    let mut srv = t!(TcpListener::bind("127.0.0.1:0").await);
-    let addr = t!(srv.local_addr());
-
-    let (server_cx, client_cx) = contexts();
-
-    let server = async move {
-        let mut incoming = srv.incoming();
-        let socket = t!(incoming.next().await.unwrap());
-        let socket = t!(server_cx.accept(socket).await);
-        copy_data(socket).await
-    };
-
-    let client = async move {
-        let socket = t!(TcpStream::connect(&addr).await);
-        let mut socket = t!(client_cx.connect("localhost", socket).await);
-        let mut data = Vec::new();
-        t!(socket.read_to_end(&mut data).await);
-        data
-    };
-
-    // Finally, run everything!
-    let (_, data) = join!(server, client);
-    // assert_eq!(amt, AMT);
-    assert!(data == vec![9; AMT]);
-}
-
-#[tokio::test]
-async fn one_byte_at_a_time() {
-    const AMT: usize = 1024;
-    drop(env_logger::try_init());
-
-    let mut srv = t!(TcpListener::bind("127.0.0.1:0").await);
-    let addr = t!(srv.local_addr());
-
-    let (server_cx, client_cx) = contexts();
-
-    let server = async move {
-        let mut incoming = srv.incoming();
-        let socket = t!(incoming.next().await.unwrap());
-        let mut socket = t!(server_cx.accept(socket).await);
-        let mut amt = 0;
-        for b in std::iter::repeat(9).take(AMT) {
-            let data = [b as u8];
-            t!(socket.write_all(&data).await);
-            amt += 1;
-        }
-        amt
-    };
-
-    let client = async move {
-        let socket = t!(TcpStream::connect(&addr).await);
-        let mut socket = t!(client_cx.connect("localhost", socket).await);
-        let mut data = Vec::new();
-        loop {
-            let mut buf = [0; 1];
-            match socket.read_exact(&mut buf).await {
-                Ok(_) => data.extend_from_slice(&buf),
-                Err(ref err) if err.kind() == ErrorKind::UnexpectedEof => break,
-                Err(err) => panic!(err),
-            }
-        }
-        data
-    };
-
-    let (amt, data) = join!(server, client);
-    assert_eq!(amt, AMT);
-    assert!(data == vec![9; AMT as usize]);
-}

tokio-util/CHANGELOG.md

@@ -1,3 +1,32 @@
+# 0.3.1 (March 18, 2020)
+
+### Fixed
+
+- Adjust minimum-supported Tokio version to v0.2.5 to account for an internal
+  dependency on features in that version of Tokio. ([#2326])
+
+# 0.3.0 (March 4, 2020)
+
+### Changed
+
+- **Breaking Change**: Change `Encoder` trait to take a generic `Item` parameter, which allows
+  codec writers to pass references into `Framed` and `FramedWrite` types. ([#1746])
+
+### Added
+
+- Add futures-io/tokio::io compatibility layer. ([#2117])
+- Add `Framed::with_capacity`. ([#2215])
+
+### Fixed
+
+- Use advance over split_to when data is not needed. ([#2198])
+
 # 0.2.0 (November 26, 2019)
 
 - Initial release
+
+[#2326]: https://github.com/tokio-rs/tokio/pull/2326
+[#2215]: https://github.com/tokio-rs/tokio/pull/2215
+[#2198]: https://github.com/tokio-rs/tokio/pull/2198
+[#2117]: https://github.com/tokio-rs/tokio/pull/2117
+[#1746]: https://github.com/tokio-rs/tokio/pull/1746

tokio-util/Cargo.toml

@@ -7,13 +7,13 @@ name = "tokio-util"
 #   - Cargo.toml
 # - Update CHANGELOG.md.
 # - Create "v0.2.x" git tag.
-version = "0.2.0"
+version = "0.3.1"
 edition = "2018"
 authors = ["Tokio Contributors <[email protected]>"]
 license = "MIT"
 repository = "https://github.com/tokio-rs/tokio"
 homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio-util/0.2.0/tokio_util"
+documentation = "https://docs.rs/tokio-util/0.3.1/tokio_util"
 description = """
 Additional utilities for working with Tokio.
 """
@@ -24,19 +24,21 @@ categories = ["asynchronous"]
 default = []
 
 # Shorthand for enabling everything
-full = ["codec", "udp"]
+full = ["codec", "udp", "compat"]
 
+compat = ["futures-io",]
 codec = ["tokio/stream"]
 udp = ["tokio/udp"]
 
 [dependencies]
-tokio = { version = "0.2.0", path = "../tokio" }
+tokio = { version = "0.2.5", path = "../tokio" }
 
 bytes = "0.5.0"
 futures-core = "0.3.0"
 futures-sink = "0.3.0"
+futures-io = { version = "0.3.0", optional = true }
 log = "0.4"
-pin-project-lite = "0.1.1"
+pin-project-lite = "0.1.4"
 
 [dev-dependencies]
 tokio = { version = "0.2.0", path = "../tokio", features = ["full"] }

tokio-util/src/cfg.rs

@@ -8,6 +8,16 @@ macro_rules! cfg_codec {
     }
 }
 
+macro_rules! cfg_compat {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "compat")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "compat")))]
+            $item
+        )*
+    }
+}
+
 macro_rules! cfg_udp {
     ($($item:item)*) => {
         $(

tokio-util/src/codec/bytes_codec.rs

@@ -4,7 +4,43 @@ use crate::codec::encoder::Encoder;
 use bytes::{BufMut, Bytes, BytesMut};
 use std::io;
 
-/// A simple `Codec` implementation that just ships bytes around.
+/// A simple [`Decoder`] and [`Encoder`] implementation that just ships bytes around.
+///
+/// [`Decoder`]: crate::codec::Decoder
+/// [`Encoder`]: crate::codec::Encoder
+///
+/// # Example
+///
+/// Turn an [`AsyncRead`] into a stream of `Result<`[`BytesMut`]`, `[`Error`]`>`.
+///
+/// [`AsyncRead`]: tokio::io::AsyncRead
+/// [`BytesMut`]: bytes::BytesMut
+/// [`Error`]: std::io::Error
+///
+/// ```
+/// # mod hidden {
+/// # #[allow(unused_imports)]
+/// use tokio::fs::File;
+/// # }
+/// use tokio::io::AsyncRead;
+/// use tokio_util::codec::{FramedRead, BytesCodec};
+///
+/// # enum File {}
+/// # impl File {
+/// #     async fn open(_name: &str) -> Result<impl AsyncRead, std::io::Error> {
+/// #         use std::io::Cursor;
+/// #         Ok(Cursor::new(vec![0, 1, 2, 3, 4, 5]))
+/// #     }
+/// # }
+/// #
+/// # #[tokio::main(core_threads = 1)]
+/// # async fn main() -> Result<(), std::io::Error> {
+/// let my_async_read = File::open("filename.txt").await?;
+/// let my_stream_of_bytes = FramedRead::new(my_async_read, BytesCodec::new());
+/// # Ok(())
+/// # }
+/// ```
+///
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)]
 pub struct BytesCodec(());
 
@@ -29,8 +65,7 @@ impl Decoder for BytesCodec {
     }
 }
 
-impl Encoder for BytesCodec {
-    type Item = Bytes;
+impl Encoder<Bytes> for BytesCodec {
     type Error = io::Error;
 
     fn encode(&mut self, data: Bytes, buf: &mut BytesMut) -> Result<(), io::Error> {

tokio-util/src/codec/decoder.rs

@@ -1,4 +1,3 @@
-use crate::codec::encoder::Encoder;
 use crate::codec::Framed;
 
 use tokio::io::{AsyncRead, AsyncWrite};
@@ -8,14 +7,17 @@ use std::io;
 
 /// Decoding of frames via buffers.
 ///
-/// This trait is used when constructing an instance of `Framed` or
-/// `FramedRead`. An implementation of `Decoder` takes a byte stream that has
+/// This trait is used when constructing an instance of [`Framed`] or
+/// [`FramedRead`]. An implementation of `Decoder` takes a byte stream that has
 /// already been buffered in `src` and decodes the data into a stream of
 /// `Self::Item` frames.
 ///
 /// Implementations are able to track state on `self`, which enables
 /// implementing stateful streaming parsers. In many cases, though, this type
 /// will simply be a unit struct (e.g. `struct HttpDecoder`).
+///
+/// [`Framed`]: crate::codec::Framed
+/// [`FramedRead`]: crate::codec::FramedRead
 pub trait Decoder {
     /// The type of decoded frames.
     type Item;
@@ -27,17 +29,19 @@ pub trait Decoder {
     /// useful to report the failure as an `Item`.
     ///
     /// `From<io::Error>` is required in the interest of making `Error` suitable
-    /// for returning directly from a `FramedRead`, and to enable the default
+    /// for returning directly from a [`FramedRead`], and to enable the default
     /// implementation of `decode_eof` to yield an `io::Error` when the decoder
     /// fails to consume all available data.
     ///
     /// Note that implementors of this trait can simply indicate `type Error =
     /// io::Error` to use I/O errors as this type.
+    ///
+    /// [`FramedRead`]: crate::codec::FramedRead
     type Error: From<io::Error>;
 
     /// Attempts to decode a frame from the provided buffer of bytes.
     ///
-    /// This method is called by `FramedRead` whenever bytes are ready to be
+    /// This method is called by [`FramedRead`] whenever bytes are ready to be
     /// parsed. The provided buffer of bytes is what's been read so far, and
     /// this instance of `Decode` can determine whether an entire frame is in
     /// the buffer and is ready to be returned.
@@ -49,7 +53,7 @@ pub trait Decoder {
     /// most circumstances.
     ///
     /// If the bytes look valid, but a frame isn't fully available yet, then
-    /// `Ok(None)` is returned. This indicates to the `Framed` instance that
+    /// `Ok(None)` is returned. This indicates to the [`Framed`] instance that
     /// it needs to read some more bytes before calling this method again.
     ///
     /// Note that the bytes provided may be empty. If a previous call to
@@ -58,9 +62,12 @@ pub trait Decoder {
     /// be read.
     ///
     /// Finally, if the bytes in the buffer are malformed then an error is
-    /// returned indicating why. This informs `Framed` that the stream is now
+    /// returned indicating why. This informs [`Framed`] that the stream is now
     /// corrupt and should be terminated.
     ///
+    /// [`Framed`]: crate::codec::Framed
+    /// [`FramedRead`]: crate::codec::FramedRead
+    ///
     /// # Buffer management
     ///
     /// Before returning from the function, implementations should ensure that
@@ -128,7 +135,7 @@ pub trait Decoder {
         }
     }
 
-    /// Provides a `Stream` and `Sink` interface for reading and writing to this
+    /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this
     /// `Io` object, using `Decode` and `Encode` to read and write the raw data.
     ///
     /// Raw I/O objects work with byte sequences, but higher-level code usually
@@ -143,11 +150,15 @@ pub trait Decoder {
     /// underlying object.
     ///
     /// If you want to work more directly with the streams and sink, consider
-    /// calling `split` on the `Framed` returned by this method, which will
+    /// calling `split` on the [`Framed`] returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`Sink`]: futures_sink::Sink
+    /// [`Framed`]: crate::codec::Framed
     fn framed<T: AsyncRead + AsyncWrite + Sized>(self, io: T) -> Framed<T, Self>
     where
-        Self: Encoder + Sized,
+        Self: Sized,
     {
         Framed::new(io, self)
     }

tokio-util/src/codec/encoder.rs

@@ -2,21 +2,24 @@ use bytes::BytesMut;
 use std::io;
 
 /// Trait of helper objects to write out messages as bytes, for use with
-/// `FramedWrite`.
-pub trait Encoder {
-    /// The type of items consumed by the `Encoder`
-    type Item;
-
+/// [`FramedWrite`].
+///
+/// [`FramedWrite`]: crate::codec::FramedWrite
+pub trait Encoder<Item> {
     /// The type of encoding errors.
     ///
-    /// `FramedWrite` requires `Encoder`s errors to implement `From<io::Error>`
+    /// [`FramedWrite`] requires `Encoder`s errors to implement `From<io::Error>`
     /// in the interest letting it return `Error`s directly.
+    ///
+    /// [`FramedWrite`]: crate::codec::FramedWrite
     type Error: From<io::Error>;
 
     /// Encodes a frame into the buffer provided.
     ///
     /// This method will encode `item` into the byte buffer provided by `dst`.
-    /// The `dst` provided is an internal buffer of the `Framed` instance and
+    /// The `dst` provided is an internal buffer of the [`FramedWrite`] instance and
     /// will be written out when possible.
-    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error>;
+    ///
+    /// [`FramedWrite`]: crate::codec::FramedWrite
+    fn encode(&mut self, item: Item, dst: &mut BytesMut) -> Result<(), Self::Error>;
 }

tokio-util/src/codec/framed.rs

@@ -1,92 +1,119 @@
 use crate::codec::decoder::Decoder;
 use crate::codec::encoder::Encoder;
-use crate::codec::framed_read::{framed_read2, framed_read2_with_buffer, FramedRead2};
-use crate::codec::framed_write::{framed_write2, framed_write2_with_buffer, FramedWrite2};
+use crate::codec::framed_impl::{FramedImpl, RWFrames, ReadFrame, WriteFrame};
 
-use tokio::{io::{AsyncBufRead, AsyncRead, AsyncWrite}, stream::Stream};
+use tokio::{
+    io::{AsyncRead, AsyncWrite},
+    stream::Stream,
+};
 
 use bytes::BytesMut;
 use futures_sink::Sink;
 use pin_project_lite::pin_project;
 use std::fmt;
-use std::io::{self, BufRead, Read, Write};
-use std::mem::MaybeUninit;
+use std::io;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
 pin_project! {
-    /// A unified `Stream` and `Sink` interface to an underlying I/O object, using
+    /// A unified [`Stream`] and [`Sink`] interface to an underlying I/O object, using
     /// the `Encoder` and `Decoder` traits to encode and decode frames.
     ///
-    /// You can create a `Framed` instance by using the `AsyncRead::framed` adapter.
+    /// You can create a `Framed` instance by using the [`Decoder::framed`] adapter, or
+    /// by using the `new` function seen below.
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`Sink`]: futures_sink::Sink
+    /// [`AsyncRead`]: tokio::io::AsyncRead
+    /// [`Decoder::framed`]: crate::codec::Decoder::framed()
     pub struct Framed<T, U> {
         #[pin]
-        inner: FramedRead2<FramedWrite2<Fuse<T, U>>>,
-    }
-}
-
-pin_project! {
-    pub(crate) struct Fuse<T, U> {
-        #[pin]
-        pub(crate) io: T,
-        pub(crate) codec: U,
-    }
-}
-
-/// Abstracts over `FramedRead2` being either `FramedRead2<FramedWrite2<Fuse<T, U>>>` or
-/// `FramedRead2<Fuse<T, U>>` and lets the io and codec parts be extracted in either case.
-pub(crate) trait ProjectFuse {
-    type Io;
-    type Codec;
-
-    fn project(self: Pin<&mut Self>) -> Fuse<Pin<&mut Self::Io>, &mut Self::Codec>;
-}
-
-impl<T, U> ProjectFuse for Fuse<T, U> {
-    type Io = T;
-    type Codec = U;
-
-    fn project(self: Pin<&mut Self>) -> Fuse<Pin<&mut Self::Io>, &mut Self::Codec> {
-        let self_ = self.project();
-        Fuse {
-            io: self_.io,
-            codec: self_.codec,
-        }
+        inner: FramedImpl<T, U, RWFrames>
     }
 }
 
 impl<T, U> Framed<T, U>
 where
     T: AsyncRead + AsyncWrite,
-    U: Decoder + Encoder,
 {
-    /// Provides a `Stream` and `Sink` interface for reading and writing to this
-    /// `Io` object, using `Decode` and `Encode` to read and write the raw data.
+    /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this
+    /// I/O object, using [`Decoder`] and [`Encoder`] to read and write the raw data.
     ///
     /// Raw I/O objects work with byte sequences, but higher-level code usually
     /// wants to batch these into meaningful chunks, called "frames". This
-    /// method layers framing on top of an I/O object, by using the `Codec`
+    /// method layers framing on top of an I/O object, by using the codec
     /// traits to handle encoding and decoding of messages frames. Note that
     /// the incoming and outgoing frame types may be distinct.
     ///
-    /// This function returns a *single* object that is both `Stream` and
-    /// `Sink`; grouping this into a single object is often useful for layering
+    /// This function returns a *single* object that is both [`Stream`] and
+    /// [`Sink`]; grouping this into a single object is often useful for layering
     /// things like gzip or TLS, which require both read and write access to the
     /// underlying object.
     ///
     /// If you want to work more directly with the streams and sink, consider
-    /// calling `split` on the `Framed` returned by this method, which will
+    /// calling [`split`] on the `Framed` returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`Sink`]: futures_sink::Sink
+    /// [`Decode`]: crate::codec::Decoder
+    /// [`Encoder`]: crate::codec::Encoder
+    /// [`split`]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html#method.split
     pub fn new(inner: T, codec: U) -> Framed<T, U> {
         Framed {
-            inner: framed_read2(framed_write2(Fuse { io: inner, codec })),
+            inner: FramedImpl {
+                inner,
+                codec,
+                state: Default::default(),
+            },
+        }
+    }
+
+    /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this
+    /// I/O object, using [`Decoder`] and [`Encoder`] to read and write the raw data,
+    /// with a specific read buffer initial capacity.
+    ///
+    /// Raw I/O objects work with byte sequences, but higher-level code usually
+    /// wants to batch these into meaningful chunks, called "frames". This
+    /// method layers framing on top of an I/O object, by using the codec
+    /// traits to handle encoding and decoding of messages frames. Note that
+    /// the incoming and outgoing frame types may be distinct.
+    ///
+    /// This function returns a *single* object that is both [`Stream`] and
+    /// [`Sink`]; grouping this into a single object is often useful for layering
+    /// things like gzip or TLS, which require both read and write access to the
+    /// underlying object.
+    ///
+    /// If you want to work more directly with the streams and sink, consider
+    /// calling [`split`] on the `Framed` returned by this method, which will
+    /// break them into separate objects, allowing them to interact more easily.
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`Sink`]: futures_sink::Sink
+    /// [`Decode`]: crate::codec::Decoder
+    /// [`Encoder`]: crate::codec::Encoder
+    /// [`split`]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html#method.split
+    pub fn with_capacity(inner: T, codec: U, capacity: usize) -> Framed<T, U> {
+        Framed {
+            inner: FramedImpl {
+                inner,
+                codec,
+                state: RWFrames {
+                    read: ReadFrame {
+                        eof: false,
+                        is_readable: false,
+                        buffer: BytesMut::with_capacity(capacity),
+                    },
+                    write: WriteFrame::default(),
+                },
+            },
         }
     }
 }
 
 impl<T, U> Framed<T, U> {
-    /// Provides a `Stream` and `Sink` interface for reading and writing to this
-    /// `Io` object, using `Decode` and `Encode` to read and write the raw data.
+    /// Provides a [`Stream`] and [`Sink`] interface for reading and writing to this
+    /// I/O object, using [`Decoder`] and [`Encoder`] to read and write the raw data.
     ///
     /// Raw I/O objects work with byte sequences, but higher-level code usually
     /// wants to batch these into meaningful chunks, called "frames". This
@@ -94,104 +121,112 @@ impl<T, U> Framed<T, U> {
     /// traits to handle encoding and decoding of messages frames. Note that
     /// the incoming and outgoing frame types may be distinct.
     ///
-    /// This function returns a *single* object that is both `Stream` and
-    /// `Sink`; grouping this into a single object is often useful for layering
+    /// This function returns a *single* object that is both [`Stream`] and
+    /// [`Sink`]; grouping this into a single object is often useful for layering
     /// things like gzip or TLS, which require both read and write access to the
     /// underlying object.
     ///
     /// This objects takes a stream and a readbuffer and a writebuffer. These field
-    /// can be obtained from an existing `Framed` with the `into_parts` method.
+    /// can be obtained from an existing `Framed` with the [`into_parts`] method.
     ///
     /// If you want to work more directly with the streams and sink, consider
-    /// calling `split` on the `Framed` returned by this method, which will
+    /// calling [`split`] on the `Framed` returned by this method, which will
     /// break them into separate objects, allowing them to interact more easily.
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`Sink`]: futures_sink::Sink
+    /// [`Decoder`]: crate::codec::Decoder
+    /// [`Encoder`]: crate::codec::Encoder
+    /// [`into_parts`]: crate::codec::Framed::into_parts()
+    /// [`split`]: https://docs.rs/futures/0.3/futures/stream/trait.StreamExt.html#method.split
     pub fn from_parts(parts: FramedParts<T, U>) -> Framed<T, U> {
         Framed {
-            inner: framed_read2_with_buffer(
-                framed_write2_with_buffer(
-                    Fuse {
-                        io: parts.io,
-                        codec: parts.codec,
-                    },
-                    parts.write_buf,
-                ),
-                parts.read_buf,
-            ),
+            inner: FramedImpl {
+                inner: parts.io,
+                codec: parts.codec,
+                state: RWFrames {
+                    read: parts.read_buf.into(),
+                    write: parts.write_buf.into(),
+                },
+            },
         }
     }
 
     /// Returns a reference to the underlying I/O stream wrapped by
-    /// `Frame`.
+    /// `Framed`.
     ///
     /// Note that care should be taken to not tamper with the underlying stream
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_ref(&self) -> &T {
-        &self.inner.get_ref().get_ref().io
+        &self.inner.inner
     }
 
     /// Returns a mutable reference to the underlying I/O stream wrapped by
-    /// `Frame`.
+    /// `Framed`.
     ///
     /// Note that care should be taken to not tamper with the underlying stream
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_mut(&mut self) -> &mut T {
-        &mut self.inner.get_mut().get_mut().io
+        &mut self.inner.inner
     }
 
     /// Returns a reference to the underlying codec wrapped by
-    /// `Frame`.
+    /// `Framed`.
     ///
     /// Note that care should be taken to not tamper with the underlying codec
     /// as it may corrupt the stream of frames otherwise being worked with.
     pub fn codec(&self) -> &U {
-        &self.inner.get_ref().get_ref().codec
+        &self.inner.codec
     }
 
     /// Returns a mutable reference to the underlying codec wrapped by
-    /// `Frame`.
+    /// `Framed`.
     ///
     /// Note that care should be taken to not tamper with the underlying codec
     /// as it may corrupt the stream of frames otherwise being worked with.
     pub fn codec_mut(&mut self) -> &mut U {
-        &mut self.inner.get_mut().get_mut().codec
+        &mut self.inner.codec
     }
 
     /// Returns a reference to the read buffer.
     pub fn read_buffer(&self) -> &BytesMut {
-        self.inner.buffer()
+        &self.inner.state.read.buffer
     }
 
-    /// Consumes the `Frame`, returning its underlying I/O stream.
+    /// Returns a mutable reference to the read buffer.
+    pub fn read_buffer_mut(&mut self) -> &mut BytesMut {
+        &mut self.inner.state.read.buffer
+    }
+
+    /// Consumes the `Framed`, returning its underlying I/O stream.
     ///
     /// Note that care should be taken to not tamper with the underlying stream
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn into_inner(self) -> T {
-        self.inner.into_inner().into_inner().io
+        self.inner.inner
     }
 
-    /// Consumes the `Frame`, returning its underlying I/O stream, the buffer
+    /// Consumes the `Framed`, returning its underlying I/O stream, the buffer
     /// with unprocessed data, and the codec.
     ///
     /// Note that care should be taken to not tamper with the underlying stream
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn into_parts(self) -> FramedParts<T, U> {
-        let (inner, read_buf) = self.inner.into_parts();
-        let (inner, write_buf) = inner.into_parts();
-
         FramedParts {
-            io: inner.io,
-            codec: inner.codec,
-            read_buf,
-            write_buf,
+            io: self.inner.inner,
+            codec: self.inner.codec,
+            read_buf: self.inner.state.read.buffer,
+            write_buf: self.inner.state.write.buffer,
             _priv: (),
         }
     }
 }
 
+// This impl just defers to the underlying FramedImpl
 impl<T, U> Stream for Framed<T, U>
 where
     T: AsyncRead,
@@ -204,28 +239,29 @@ where
     }
 }
 
+// This impl just defers to the underlying FramedImpl
 impl<T, I, U> Sink<I> for Framed<T, U>
 where
     T: AsyncWrite,
-    U: Encoder<Item = I>,
+    U: Encoder<I>,
     U::Error: From<io::Error>,
 {
     type Error = U::Error;
 
     fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project().inner.get_pin_mut().poll_ready(cx)
+        self.project().inner.poll_ready(cx)
     }
 
     fn start_send(self: Pin<&mut Self>, item: I) -> Result<(), Self::Error> {
-        self.project().inner.get_pin_mut().start_send(item)
+        self.project().inner.start_send(item)
     }
 
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project().inner.get_pin_mut().poll_flush(cx)
+        self.project().inner.poll_flush(cx)
     }
 
     fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project().inner.get_pin_mut().poll_close(cx)
+        self.project().inner.poll_close(cx)
     }
 }
 
@@ -236,108 +272,19 @@ where
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("Framed")
-            .field("io", &self.inner.get_ref().get_ref().io)
-            .field("codec", &self.inner.get_ref().get_ref().codec)
+            .field("io", self.get_ref())
+            .field("codec", self.codec())
             .finish()
     }
 }
 
-// ===== impl Fuse =====
-
-impl<T: Read, U> Read for Fuse<T, U> {
-    fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> {
-        self.io.read(dst)
-    }
-}
-
-impl<T: BufRead, U> BufRead for Fuse<T, U> {
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        self.io.fill_buf()
-    }
-
-    fn consume(&mut self, amt: usize) {
-        self.io.consume(amt)
-    }
-}
-
-impl<T: AsyncRead, U> AsyncRead for Fuse<T, U> {
-    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [MaybeUninit<u8>]) -> bool {
-        self.io.prepare_uninitialized_buffer(buf)
-    }
-
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut [u8],
-    ) -> Poll<Result<usize, io::Error>> {
-        self.project().io.poll_read(cx, buf)
-    }
-}
-
-impl<T: AsyncBufRead, U> AsyncBufRead for Fuse<T, U> {
-    fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
-        self.project().io.poll_fill_buf(cx)
-    }
-
-    fn consume(self: Pin<&mut Self>, amt: usize) {
-        self.project().io.consume(amt)
-    }
-}
-
-impl<T: Write, U> Write for Fuse<T, U> {
-    fn write(&mut self, src: &[u8]) -> io::Result<usize> {
-        self.io.write(src)
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        self.io.flush()
-    }
-}
-
-impl<T: AsyncWrite, U> AsyncWrite for Fuse<T, U> {
-    fn poll_write(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &[u8],
-    ) -> Poll<Result<usize, io::Error>> {
-        self.project().io.poll_write(cx, buf)
-    }
-
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        self.project().io.poll_flush(cx)
-    }
-
-    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        self.project().io.poll_shutdown(cx)
-    }
-}
-
-impl<T, U: Decoder> Decoder for Fuse<T, U> {
-    type Item = U::Item;
-    type Error = U::Error;
-
-    fn decode(&mut self, buffer: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        self.codec.decode(buffer)
-    }
-
-    fn decode_eof(&mut self, buffer: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        self.codec.decode_eof(buffer)
-    }
-}
-
-impl<T, U: Encoder> Encoder for Fuse<T, U> {
-    type Item = U::Item;
-    type Error = U::Error;
-
-    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
-        self.codec.encode(item, dst)
-    }
-}
-
 /// `FramedParts` contains an export of the data of a Framed transport.
-/// It can be used to construct a new `Framed` with a different codec.
+/// It can be used to construct a new [`Framed`] with a different codec.
 /// It contains all current buffers and the inner transport.
+///
+/// [`Framed`]: crate::codec::Framed
 #[derive(Debug)]
+#[allow(clippy::manual_non_exhaustive)]
 pub struct FramedParts<T, U> {
     /// The inner transport used to read bytes to and write bytes to
     pub io: T,
@@ -358,7 +305,10 @@ pub struct FramedParts<T, U> {
 
 impl<T, U> FramedParts<T, U> {
     /// Create a new, default, `FramedParts`
-    pub fn new(io: T, codec: U) -> FramedParts<T, U> {
+    pub fn new<I>(io: T, codec: U) -> FramedParts<T, U>
+    where
+        U: Encoder<I>,
+    {
         FramedParts {
             io,
             codec,

tokio-util/src/codec/framed_impl.rs

@@ -0,0 +1,225 @@
+use crate::codec::decoder::Decoder;
+use crate::codec::encoder::Encoder;
+
+use tokio::{
+    io::{AsyncRead, AsyncWrite},
+    stream::Stream,
+};
+
+use bytes::{Buf, BytesMut};
+use futures_core::ready;
+use futures_sink::Sink;
+use log::trace;
+use pin_project_lite::pin_project;
+use std::borrow::{Borrow, BorrowMut};
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+pin_project! {
+    #[derive(Debug)]
+    pub(crate) struct FramedImpl<T, U, State> {
+        #[pin]
+        pub(crate) inner: T,
+        pub(crate) state: State,
+        pub(crate) codec: U,
+    }
+}
+
+const INITIAL_CAPACITY: usize = 8 * 1024;
+const BACKPRESSURE_BOUNDARY: usize = INITIAL_CAPACITY;
+
+pub(crate) struct ReadFrame {
+    pub(crate) eof: bool,
+    pub(crate) is_readable: bool,
+    pub(crate) buffer: BytesMut,
+}
+
+pub(crate) struct WriteFrame {
+    pub(crate) buffer: BytesMut,
+}
+
+#[derive(Default)]
+pub(crate) struct RWFrames {
+    pub(crate) read: ReadFrame,
+    pub(crate) write: WriteFrame,
+}
+
+impl Default for ReadFrame {
+    fn default() -> Self {
+        Self {
+            eof: false,
+            is_readable: false,
+            buffer: BytesMut::with_capacity(INITIAL_CAPACITY),
+        }
+    }
+}
+
+impl Default for WriteFrame {
+    fn default() -> Self {
+        Self {
+            buffer: BytesMut::with_capacity(INITIAL_CAPACITY),
+        }
+    }
+}
+
+impl From<BytesMut> for ReadFrame {
+    fn from(mut buffer: BytesMut) -> Self {
+        let size = buffer.capacity();
+        if size < INITIAL_CAPACITY {
+            buffer.reserve(INITIAL_CAPACITY - size);
+        }
+
+        Self {
+            buffer,
+            is_readable: size > 0,
+            eof: false,
+        }
+    }
+}
+
+impl From<BytesMut> for WriteFrame {
+    fn from(mut buffer: BytesMut) -> Self {
+        let size = buffer.capacity();
+        if size < INITIAL_CAPACITY {
+            buffer.reserve(INITIAL_CAPACITY - size);
+        }
+
+        Self { buffer }
+    }
+}
+
+impl Borrow<ReadFrame> for RWFrames {
+    fn borrow(&self) -> &ReadFrame {
+        &self.read
+    }
+}
+impl BorrowMut<ReadFrame> for RWFrames {
+    fn borrow_mut(&mut self) -> &mut ReadFrame {
+        &mut self.read
+    }
+}
+impl Borrow<WriteFrame> for RWFrames {
+    fn borrow(&self) -> &WriteFrame {
+        &self.write
+    }
+}
+impl BorrowMut<WriteFrame> for RWFrames {
+    fn borrow_mut(&mut self) -> &mut WriteFrame {
+        &mut self.write
+    }
+}
+impl<T, U, R> Stream for FramedImpl<T, U, R>
+where
+    T: AsyncRead,
+    U: Decoder,
+    R: BorrowMut<ReadFrame>,
+{
+    type Item = Result<U::Item, U::Error>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let mut pinned = self.project();
+        let state: &mut ReadFrame = pinned.state.borrow_mut();
+        loop {
+            // Repeatedly call `decode` or `decode_eof` as long as it is
+            // "readable". Readable is defined as not having returned `None`. If
+            // the upstream has returned EOF, and the decoder is no longer
+            // readable, it can be assumed that the decoder will never become
+            // readable again, at which point the stream is terminated.
+            if state.is_readable {
+                if state.eof {
+                    let frame = pinned.codec.decode_eof(&mut state.buffer)?;
+                    return Poll::Ready(frame.map(Ok));
+                }
+
+                trace!("attempting to decode a frame");
+
+                if let Some(frame) = pinned.codec.decode(&mut state.buffer)? {
+                    trace!("frame decoded from buffer");
+                    return Poll::Ready(Some(Ok(frame)));
+                }
+
+                state.is_readable = false;
+            }
+
+            assert!(!state.eof);
+
+            // Otherwise, try to read more data and try again. Make sure we've
+            // got room for at least one byte to read to ensure that we don't
+            // get a spurious 0 that looks like EOF
+            state.buffer.reserve(1);
+            let bytect = match pinned.inner.as_mut().poll_read_buf(cx, &mut state.buffer)? {
+                Poll::Ready(ct) => ct,
+                Poll::Pending => return Poll::Pending,
+            };
+            if bytect == 0 {
+                state.eof = true;
+            }
+
+            state.is_readable = true;
+        }
+    }
+}
+
+impl<T, I, U, W> Sink<I> for FramedImpl<T, U, W>
+where
+    T: AsyncWrite,
+    U: Encoder<I>,
+    U::Error: From<io::Error>,
+    W: BorrowMut<WriteFrame>,
+{
+    type Error = U::Error;
+
+    fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        if self.state.borrow().buffer.len() >= BACKPRESSURE_BOUNDARY {
+            self.as_mut().poll_flush(cx)
+        } else {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    fn start_send(self: Pin<&mut Self>, item: I) -> Result<(), Self::Error> {
+        let pinned = self.project();
+        pinned
+            .codec
+            .encode(item, &mut pinned.state.borrow_mut().buffer)?;
+        Ok(())
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        trace!("flushing framed transport");
+        let mut pinned = self.project();
+
+        while !pinned.state.borrow_mut().buffer.is_empty() {
+            let WriteFrame { buffer } = pinned.state.borrow_mut();
+            trace!("writing; remaining={}", buffer.len());
+
+            let buf = &buffer;
+            let n = ready!(pinned.inner.as_mut().poll_write(cx, &buf))?;
+
+            if n == 0 {
+                return Poll::Ready(Err(io::Error::new(
+                    io::ErrorKind::WriteZero,
+                    "failed to \
+                     write frame to transport",
+                )
+                .into()));
+            }
+
+            pinned.state.borrow_mut().buffer.advance(n);
+        }
+
+        // Try flushing the underlying IO
+        ready!(pinned.inner.poll_flush(cx))?;
+
+        trace!("framed transport flushed");
+        Poll::Ready(Ok(()))
+    }
+
+    fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        ready!(self.as_mut().poll_flush(cx))?;
+        ready!(self.project().inner.poll_shutdown(cx))?;
+
+        Poll::Ready(Ok(()))
+    }
+}

tokio-util/src/codec/framed_read.rs

@@ -1,36 +1,26 @@
-use crate::codec::framed::{Fuse, ProjectFuse};
+use crate::codec::framed_impl::{FramedImpl, ReadFrame};
 use crate::codec::Decoder;
 
 use tokio::{io::AsyncRead, stream::Stream};
 
 use bytes::BytesMut;
 use futures_sink::Sink;
-use log::trace;
 use pin_project_lite::pin_project;
 use std::fmt;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
 pin_project! {
-    /// A `Stream` of messages decoded from an `AsyncRead`.
+    /// A [`Stream`] of messages decoded from an [`AsyncRead`].
+    ///
+    /// [`Stream`]: tokio::stream::Stream
+    /// [`AsyncRead`]: tokio::io::AsyncRead
     pub struct FramedRead<T, D> {
         #[pin]
-        inner: FramedRead2<Fuse<T, D>>,
+        inner: FramedImpl<T, D, ReadFrame>,
     }
 }
 
-pin_project! {
-    pub(crate) struct FramedRead2<T> {
-        #[pin]
-        inner: T,
-        eof: bool,
-        is_readable: bool,
-        buffer: BytesMut,
-    }
-}
-
-const INITIAL_CAPACITY: usize = 8 * 1024;
-
 // ===== impl FramedRead =====
 
 impl<T, D> FramedRead<T, D>
@@ -41,10 +31,27 @@ where
     /// Creates a new `FramedRead` with the given `decoder`.
     pub fn new(inner: T, decoder: D) -> FramedRead<T, D> {
         FramedRead {
-            inner: framed_read2(Fuse {
-                io: inner,
+            inner: FramedImpl {
+                inner,
                 codec: decoder,
-            }),
+                state: Default::default(),
+            },
+        }
+    }
+
+    /// Creates a new `FramedRead` with the given `decoder` and a buffer of `capacity`
+    /// initial size.
+    pub fn with_capacity(inner: T, decoder: D, capacity: usize) -> FramedRead<T, D> {
+        FramedRead {
+            inner: FramedImpl {
+                inner,
+                codec: decoder,
+                state: ReadFrame {
+                    eof: false,
+                    is_readable: false,
+                    buffer: BytesMut::with_capacity(capacity),
+                },
+            },
         }
     }
 }
@@ -57,7 +64,7 @@ impl<T, D> FramedRead<T, D> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_ref(&self) -> &T {
-        &self.inner.inner.io
+        &self.inner.inner
     }
 
     /// Returns a mutable reference to the underlying I/O stream wrapped by
@@ -67,7 +74,7 @@ impl<T, D> FramedRead<T, D> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_mut(&mut self) -> &mut T {
-        &mut self.inner.inner.io
+        &mut self.inner.inner
     }
 
     /// Consumes the `FramedRead`, returning its underlying I/O stream.
@@ -76,25 +83,26 @@ impl<T, D> FramedRead<T, D> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn into_inner(self) -> T {
-        self.inner.inner.io
+        self.inner.inner
     }
 
     /// Returns a reference to the underlying decoder.
     pub fn decoder(&self) -> &D {
-        &self.inner.inner.codec
+        &self.inner.codec
     }
 
     /// Returns a mutable reference to the underlying decoder.
     pub fn decoder_mut(&mut self) -> &mut D {
-        &mut self.inner.inner.codec
+        &mut self.inner.codec
     }
 
     /// Returns a reference to the read buffer.
     pub fn read_buffer(&self) -> &BytesMut {
-        &self.inner.buffer
+        &self.inner.state.buffer
     }
 }
 
+// This impl just defers to the underlying FramedImpl
 impl<T, D> Stream for FramedRead<T, D>
 where
     T: AsyncRead,
@@ -115,43 +123,19 @@ where
     type Error = T::Error;
 
     fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project()
-            .inner
-            .project()
-            .inner
-            .project()
-            .io
-            .poll_ready(cx)
+        self.project().inner.project().inner.poll_ready(cx)
     }
 
     fn start_send(self: Pin<&mut Self>, item: I) -> Result<(), Self::Error> {
-        self.project()
-            .inner
-            .project()
-            .inner
-            .project()
-            .io
-            .start_send(item)
+        self.project().inner.project().inner.start_send(item)
     }
 
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project()
-            .inner
-            .project()
-            .inner
-            .project()
-            .io
-            .poll_flush(cx)
+        self.project().inner.project().inner.poll_flush(cx)
     }
 
     fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.project()
-            .inner
-            .project()
-            .inner
-            .project()
-            .io
-            .poll_close(cx)
+        self.project().inner.project().inner.poll_close(cx)
     }
 }
 
@@ -162,126 +146,11 @@ where
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("FramedRead")
-            .field("inner", &self.inner.inner.io)
-            .field("decoder", &self.inner.inner.codec)
-            .field("eof", &self.inner.eof)
-            .field("is_readable", &self.inner.is_readable)
-            .field("buffer", &self.inner.buffer)
+            .field("inner", &self.get_ref())
+            .field("decoder", &self.decoder())
+            .field("eof", &self.inner.state.eof)
+            .field("is_readable", &self.inner.state.is_readable)
+            .field("buffer", &self.read_buffer())
             .finish()
     }
 }
-
-// ===== impl FramedRead2 =====
-
-pub(crate) fn framed_read2<T>(inner: T) -> FramedRead2<T> {
-    FramedRead2 {
-        inner,
-        eof: false,
-        is_readable: false,
-        buffer: BytesMut::with_capacity(INITIAL_CAPACITY),
-    }
-}
-
-pub(crate) fn framed_read2_with_buffer<T>(inner: T, mut buf: BytesMut) -> FramedRead2<T> {
-    if buf.capacity() < INITIAL_CAPACITY {
-        let bytes_to_reserve = INITIAL_CAPACITY - buf.capacity();
-        buf.reserve(bytes_to_reserve);
-    }
-    FramedRead2 {
-        inner,
-        eof: false,
-        is_readable: !buf.is_empty(),
-        buffer: buf,
-    }
-}
-
-impl<T> FramedRead2<T> {
-    pub(crate) fn get_ref(&self) -> &T {
-        &self.inner
-    }
-
-    pub(crate) fn into_inner(self) -> T {
-        self.inner
-    }
-
-    pub(crate) fn into_parts(self) -> (T, BytesMut) {
-        (self.inner, self.buffer)
-    }
-
-    pub(crate) fn get_mut(&mut self) -> &mut T {
-        &mut self.inner
-    }
-
-    pub(crate) fn get_pin_mut(self: Pin<&mut Self>) -> Pin<&mut T> {
-        self.project().inner
-    }
-
-    pub(crate) fn buffer(&self) -> &BytesMut {
-        &self.buffer
-    }
-}
-
-impl<T> Stream for FramedRead2<T>
-where
-    T: ProjectFuse + AsyncRead,
-    T::Codec: Decoder,
-{
-    type Item = Result<<T::Codec as Decoder>::Item, <T::Codec as Decoder>::Error>;
-
-    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        let mut pinned = self.project();
-        loop {
-            // Repeatedly call `decode` or `decode_eof` as long as it is
-            // "readable". Readable is defined as not having returned `None`. If
-            // the upstream has returned EOF, and the decoder is no longer
-            // readable, it can be assumed that the decoder will never become
-            // readable again, at which point the stream is terminated.
-            if *pinned.is_readable {
-                if *pinned.eof {
-                    let frame = pinned
-                        .inner
-                        .as_mut()
-                        .project()
-                        .codec
-                        .decode_eof(&mut pinned.buffer)?;
-                    return Poll::Ready(frame.map(Ok));
-                }
-
-                trace!("attempting to decode a frame");
-
-                if let Some(frame) = pinned
-                    .inner
-                    .as_mut()
-                    .project()
-                    .codec
-                    .decode(&mut pinned.buffer)?
-                {
-                    trace!("frame decoded from buffer");
-                    return Poll::Ready(Some(Ok(frame)));
-                }
-
-                *pinned.is_readable = false;
-            }
-
-            assert!(!*pinned.eof);
-
-            // Otherwise, try to read more data and try again. Make sure we've
-            // got room for at least one byte to read to ensure that we don't
-            // get a spurious 0 that looks like EOF
-            pinned.buffer.reserve(1);
-            let bytect = match pinned
-                .inner
-                .as_mut()
-                .poll_read_buf(cx, &mut pinned.buffer)?
-            {
-                Poll::Ready(ct) => ct,
-                Poll::Pending => return Poll::Pending,
-            };
-            if bytect == 0 {
-                *pinned.eof = true;
-            }
-
-            *pinned.is_readable = true;
-        }
-    }
-}

tokio-util/src/codec/framed_write.rs

@@ -1,51 +1,37 @@
-use crate::codec::decoder::Decoder;
 use crate::codec::encoder::Encoder;
-use crate::codec::framed::{Fuse, ProjectFuse};
+use crate::codec::framed_impl::{FramedImpl, WriteFrame};
 
-use tokio::{io::{AsyncBufRead, AsyncRead, AsyncWrite}, stream::Stream};
+use tokio::{io::AsyncWrite, stream::Stream};
 
-use bytes::BytesMut;
-use futures_core::ready;
 use futures_sink::Sink;
-use log::trace;
 use pin_project_lite::pin_project;
 use std::fmt;
-use std::io::{self, BufRead, Read};
-use std::mem::MaybeUninit;
+use std::io;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
 pin_project! {
-    /// A `Sink` of frames encoded to an `AsyncWrite`.
+    /// A [`Sink`] of frames encoded to an `AsyncWrite`.
+    ///
+    /// [`Sink`]: futures_sink::Sink
     pub struct FramedWrite<T, E> {
         #[pin]
-        inner: FramedWrite2<Fuse<T, E>>,
+        inner: FramedImpl<T, E, WriteFrame>,
     }
 }
 
-pin_project! {
-    pub(crate) struct FramedWrite2<T> {
-        #[pin]
-        inner: T,
-        buffer: BytesMut,
-    }
-}
-
-const INITIAL_CAPACITY: usize = 8 * 1024;
-const BACKPRESSURE_BOUNDARY: usize = INITIAL_CAPACITY;
-
 impl<T, E> FramedWrite<T, E>
 where
     T: AsyncWrite,
-    E: Encoder,
 {
     /// Creates a new `FramedWrite` with the given `encoder`.
     pub fn new(inner: T, encoder: E) -> FramedWrite<T, E> {
         FramedWrite {
-            inner: framed_write2(Fuse {
-                io: inner,
+            inner: FramedImpl {
+                inner,
                 codec: encoder,
-            }),
+                state: WriteFrame::default(),
+            },
         }
     }
 }
@@ -58,7 +44,7 @@ impl<T, E> FramedWrite<T, E> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_ref(&self) -> &T {
-        &self.inner.inner.io
+        &self.inner.inner
     }
 
     /// Returns a mutable reference to the underlying I/O stream wrapped by
@@ -68,7 +54,7 @@ impl<T, E> FramedWrite<T, E> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn get_mut(&mut self) -> &mut T {
-        &mut self.inner.inner.io
+        &mut self.inner.inner
     }
 
     /// Consumes the `FramedWrite`, returning its underlying I/O stream.
@@ -77,25 +63,25 @@ impl<T, E> FramedWrite<T, E> {
     /// of data coming in as it may corrupt the stream of frames otherwise
     /// being worked with.
     pub fn into_inner(self) -> T {
-        self.inner.inner.io
+        self.inner.inner
     }
 
-    /// Returns a reference to the underlying decoder.
+    /// Returns a reference to the underlying encoder.
     pub fn encoder(&self) -> &E {
-        &self.inner.inner.codec
+        &self.inner.codec
     }
 
-    /// Returns a mutable reference to the underlying decoder.
+    /// Returns a mutable reference to the underlying encoder.
     pub fn encoder_mut(&mut self) -> &mut E {
-        &mut self.inner.inner.codec
+        &mut self.inner.codec
     }
 }
 
-// This impl just defers to the underlying FramedWrite2
+// This impl just defers to the underlying FramedImpl
 impl<T, I, E> Sink<I> for FramedWrite<T, E>
 where
     T: AsyncWrite,
-    E: Encoder<Item = I>,
+    E: Encoder<I>,
     E::Error: From<io::Error>,
 {
     type Error = E::Error;
@@ -117,6 +103,7 @@ where
     }
 }
 
+// This impl just defers to the underlying T: Stream
 impl<T, D> Stream for FramedWrite<T, D>
 where
     T: Stream,
@@ -124,13 +111,7 @@ where
     type Item = T::Item;
 
     fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        self.project()
-            .inner
-            .project()
-            .inner
-            .project()
-            .io
-            .poll_next(cx)
+        self.project().inner.project().inner.poll_next(cx)
     }
 }
 
@@ -141,181 +122,9 @@ where
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("FramedWrite")
-            .field("inner", &self.inner.get_ref().io)
-            .field("encoder", &self.inner.get_ref().codec)
-            .field("buffer", &self.inner.buffer)
+            .field("inner", &self.get_ref())
+            .field("encoder", &self.encoder())
+            .field("buffer", &self.inner.state.buffer)
             .finish()
     }
 }
-
-// ===== impl FramedWrite2 =====
-
-pub(crate) fn framed_write2<T>(inner: T) -> FramedWrite2<T> {
-    FramedWrite2 {
-        inner,
-        buffer: BytesMut::with_capacity(INITIAL_CAPACITY),
-    }
-}
-
-pub(crate) fn framed_write2_with_buffer<T>(inner: T, mut buf: BytesMut) -> FramedWrite2<T> {
-    if buf.capacity() < INITIAL_CAPACITY {
-        let bytes_to_reserve = INITIAL_CAPACITY - buf.capacity();
-        buf.reserve(bytes_to_reserve);
-    }
-    FramedWrite2 { inner, buffer: buf }
-}
-
-impl<T> FramedWrite2<T> {
-    pub(crate) fn get_ref(&self) -> &T {
-        &self.inner
-    }
-
-    pub(crate) fn into_inner(self) -> T {
-        self.inner
-    }
-
-    pub(crate) fn into_parts(self) -> (T, BytesMut) {
-        (self.inner, self.buffer)
-    }
-
-    pub(crate) fn get_mut(&mut self) -> &mut T {
-        &mut self.inner
-    }
-}
-
-impl<I, T> Sink<I> for FramedWrite2<T>
-where
-    T: ProjectFuse + AsyncWrite,
-    T::Codec: Encoder<Item = I>,
-{
-    type Error = <T::Codec as Encoder>::Error;
-
-    fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        // If the buffer is already over 8KiB, then attempt to flush it. If after flushing it's
-        // *still* over 8KiB, then apply backpressure (reject the send).
-        if self.buffer.len() >= BACKPRESSURE_BOUNDARY {
-            match self.as_mut().poll_flush(cx) {
-                Poll::Pending => return Poll::Pending,
-                Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
-                Poll::Ready(Ok(())) => (),
-            };
-
-            if self.buffer.len() >= BACKPRESSURE_BOUNDARY {
-                return Poll::Pending;
-            }
-        }
-        Poll::Ready(Ok(()))
-    }
-
-    fn start_send(self: Pin<&mut Self>, item: I) -> Result<(), Self::Error> {
-        let mut pinned = self.project();
-        pinned
-            .inner
-            .project()
-            .codec
-            .encode(item, &mut pinned.buffer)?;
-        Ok(())
-    }
-
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        trace!("flushing framed transport");
-        let mut pinned = self.project();
-
-        while !pinned.buffer.is_empty() {
-            trace!("writing; remaining={}", pinned.buffer.len());
-
-            let buf = &pinned.buffer;
-            let n = ready!(pinned.inner.as_mut().poll_write(cx, &buf))?;
-
-            if n == 0 {
-                return Poll::Ready(Err(io::Error::new(
-                    io::ErrorKind::WriteZero,
-                    "failed to \
-                     write frame to transport",
-                )
-                .into()));
-            }
-
-            // TODO: Add a way to `bytes` to do this w/o returning the drained data.
-            let _ = pinned.buffer.split_to(n);
-        }
-
-        // Try flushing the underlying IO
-        ready!(pinned.inner.poll_flush(cx))?;
-
-        trace!("framed transport flushed");
-        Poll::Ready(Ok(()))
-    }
-
-    fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        ready!(self.as_mut().poll_flush(cx))?;
-        ready!(self.project().inner.poll_shutdown(cx))?;
-
-        Poll::Ready(Ok(()))
-    }
-}
-
-impl<T: Decoder> Decoder for FramedWrite2<T> {
-    type Item = T::Item;
-    type Error = T::Error;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<T::Item>, T::Error> {
-        self.inner.decode(src)
-    }
-
-    fn decode_eof(&mut self, src: &mut BytesMut) -> Result<Option<T::Item>, T::Error> {
-        self.inner.decode_eof(src)
-    }
-}
-
-impl<T: Read> Read for FramedWrite2<T> {
-    fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> {
-        self.inner.read(dst)
-    }
-}
-
-impl<T: BufRead> BufRead for FramedWrite2<T> {
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        self.inner.fill_buf()
-    }
-
-    fn consume(&mut self, amt: usize) {
-        self.inner.consume(amt)
-    }
-}
-
-impl<T: AsyncRead> AsyncRead for FramedWrite2<T> {
-    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [MaybeUninit<u8>]) -> bool {
-        self.inner.prepare_uninitialized_buffer(buf)
-    }
-
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut [u8],
-    ) -> Poll<Result<usize, io::Error>> {
-        self.project().inner.poll_read(cx, buf)
-    }
-}
-
-impl<T: AsyncBufRead> AsyncBufRead for FramedWrite2<T> {
-    fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
-        self.project().inner.poll_fill_buf(cx)
-    }
-
-    fn consume(self: Pin<&mut Self>, amt: usize) {
-        self.project().inner.consume(amt)
-    }
-}
-
-impl<T> ProjectFuse for FramedWrite2<T>
-where
-    T: ProjectFuse,
-{
-    type Io = T::Io;
-    type Codec = T::Codec;
-
-    fn project(self: Pin<&mut Self>) -> Fuse<Pin<&mut Self::Io>, &mut Self::Codec> {
-        self.project().inner.project()
-    }
-}

tokio-util/src/codec/length_delimited.rs

@@ -302,6 +302,37 @@
 //! anywhere because it already is factored into the total frame length that
 //! is read from the byte stream.
 //!
+//! ## Example 7
+//!
+//! The following will parse a 3 byte length field at offset 0 in a 4 byte
+//! frame head, excluding the 4th byte from the yielded `BytesMut`.
+//!
+//! ```
+//! # use tokio::io::AsyncRead;
+//! # use tokio_util::codec::LengthDelimitedCodec;
+//! # fn bind_read<T: AsyncRead>(io: T) {
+//! LengthDelimitedCodec::builder()
+//!     .length_field_offset(0) // default value
+//!     .length_field_length(3)
+//!     .length_adjustment(0)  // default value
+//!     .num_skip(4) // skip the first 4 bytes
+//!     .new_read(io);
+//! # }
+//! # pub fn main() {}
+//! ```
+//!
+//! The following frame will be decoded as such:
+//!
+//! ```text
+//!                  INPUT                       DECODED
+//! +------- len ------+--- Payload ---+    +--- Payload ---+
+//! | \x00\x00\x0B\xFF |  Hello world  | => |  Hello world  |
+//! +------------------+---------------+    +---------------+
+//! ```
+//!
+//! A simple example where there are unused bytes between the length field
+//! and the payload.
+//!
 //! # Encoding
 //!
 //! [`FramedWrite`] adapts an [`AsyncWrite`] into a `Sink` of [`BytesMut`],
@@ -333,13 +364,13 @@
 //! +------------+--------------+
 //! ```
 //!
-//! [`LengthDelimitedCodec::new()`]: struct.LengthDelimitedCodec.html#method.new
-//! [`FramedRead`]: struct.FramedRead.html
-//! [`FramedWrite`]: struct.FramedWrite.html
-//! [`AsyncRead`]: ../../trait.AsyncRead.html
-//! [`AsyncWrite`]: ../../trait.AsyncWrite.html
-//! [`Encoder`]: ../trait.Encoder.html
-//! [`BytesMut`]: https://docs.rs/bytes/0.4/bytes/struct.BytesMut.html
+//! [`LengthDelimitedCodec::new()`]: method@LengthDelimitedCodec::new
+//! [`FramedRead`]: struct@FramedRead
+//! [`FramedWrite`]: struct@FramedWrite
+//! [`AsyncRead`]: trait@tokio::io::AsyncRead
+//! [`AsyncWrite`]: trait@tokio::io::AsyncWrite
+//! [`Encoder`]: trait@Encoder
+//! [`BytesMut`]: bytes::BytesMut
 
 use crate::codec::{Decoder, Encoder, Framed, FramedRead, FramedWrite};
 
@@ -494,7 +525,7 @@ impl LengthDelimitedCodec {
         let num_skip = self.builder.get_num_skip();
 
         if num_skip > 0 {
-            let _ = src.split_to(num_skip);
+            src.advance(num_skip);
         }
 
         // Ensure that the buffer has enough space to read the incoming
@@ -546,12 +577,11 @@ impl Decoder for LengthDelimitedCodec {
     }
 }
 
-impl Encoder for LengthDelimitedCodec {
-    type Item = Bytes;
+impl Encoder<Bytes> for LengthDelimitedCodec {
     type Error = io::Error;
 
     fn encode(&mut self, data: Bytes, dst: &mut BytesMut) -> Result<(), io::Error> {
-        let n = (&data).remaining();
+        let n = data.len();
 
         if n > self.builder.max_frame_len {
             return Err(io::Error::new(

tokio-util/src/codec/lines_codec.rs

@@ -4,7 +4,10 @@ use crate::codec::encoder::Encoder;
 use bytes::{Buf, BufMut, BytesMut};
 use std::{cmp, fmt, io, str, usize};
 
-/// A simple `Codec` implementation that splits up data into lines.
+/// A simple [`Decoder`] and [`Encoder`] implementation that splits up data into lines.
+///
+/// [`Decoder`]: crate::codec::Decoder
+/// [`Encoder`]: crate::codec::Encoder
 #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub struct LinesCodec {
     // Stored index of the next index to examine for a `\n` character.
@@ -33,7 +36,7 @@ impl LinesCodec {
     /// of a buffered line. See the documentation for [`new_with_max_length`]
     /// for information on why this could be a potential security risk.
     ///
-    /// [`new_with_max_length`]: #method.new_with_max_length
+    /// [`new_with_max_length`]: crate::codec::LinesCodec::new_with_max_length()
     pub fn new() -> LinesCodec {
         LinesCodec {
             next_index: 0,
@@ -45,7 +48,7 @@ impl LinesCodec {
     /// Returns a `LinesCodec` with a maximum line length limit.
     ///
     /// If this is set, calls to `LinesCodec::decode` will return a
-    /// [`LengthError`] when a line exceeds the length limit. Subsequent calls
+    /// [`LinesCodecError`] when a line exceeds the length limit. Subsequent calls
     /// will discard up to `limit` bytes from that line until a newline
     /// character is reached, returning `None` until the line over the limit
     /// has been fully discarded. After that point, calls to `decode` will
@@ -59,7 +62,7 @@ impl LinesCodec {
     /// exploit this unbounded buffer by sending an unbounded amount of input
     /// without any `\n` characters, causing unbounded memory consumption.
     ///
-    /// [`LengthError`]: ../struct.LengthError
+    /// [`LinesCodecError`]: crate::codec::LinesCodecError
     pub fn new_with_max_length(max_length: usize) -> Self {
         LinesCodec {
             max_length,
@@ -179,11 +182,14 @@ impl Decoder for LinesCodec {
     }
 }
 
-impl Encoder for LinesCodec {
-    type Item = String;
+impl<T> Encoder<T> for LinesCodec
+where
+    T: AsRef<str>,
+{
     type Error = LinesCodecError;
 
-    fn encode(&mut self, line: String, buf: &mut BytesMut) -> Result<(), LinesCodecError> {
+    fn encode(&mut self, line: T, buf: &mut BytesMut) -> Result<(), LinesCodecError> {
+        let line = line.as_ref();
         buf.reserve(line.len() + 1);
         buf.put(line.as_bytes());
         buf.put_u8(b'\n');

tokio-util/src/codec/mod.rs

@@ -1,13 +1,18 @@
-//! Utilities for encoding and decoding frames.
+//! Adaptors from AsyncRead/AsyncWrite to Stream/Sink
 //!
-//! Contains adapters to go from streams of bytes, [`AsyncRead`] and
-//! [`AsyncWrite`], to framed streams implementing [`Sink`] and [`Stream`].
-//! Framed streams are also known as transports.
+//! Raw I/O objects work with byte sequences, but higher-level code
+//! usually wants to batch these into meaningful chunks, called
+//! "frames".
 //!
-//! [`AsyncRead`]: https://docs.rs/tokio/*/tokio/io/trait.AsyncRead.html
-//! [`AsyncWrite`]: https://docs.rs/tokio/*/tokio/io/trait.AsyncWrite.html
-//! [`Stream`]: https://docs.rs/tokio/*/tokio/stream/trait.Stream.html
-//! [`Sink`]: https://docs.rs/futures-sink/*/futures_sink/trait.Sink.html
+//! This module contains adapters to go from streams of bytes,
+//! [`AsyncRead`] and [`AsyncWrite`], to framed streams implementing
+//! [`Sink`] and [`Stream`].  Framed streams are also known as
+//! transports.
+//!
+//! [`AsyncRead`]: tokio::io::AsyncRead
+//! [`AsyncWrite`]: tokio::io::AsyncWrite
+//! [`Stream`]: tokio::stream::Stream
+//! [`Sink`]: futures_sink::Sink
 
 mod bytes_codec;
 pub use self::bytes_codec::BytesCodec;
@@ -18,6 +23,10 @@ pub use self::decoder::Decoder;
 mod encoder;
 pub use self::encoder::Encoder;
 
+mod framed_impl;
+#[allow(unused_imports)]
+pub(crate) use self::framed_impl::{FramedImpl, RWFrames, ReadFrame, WriteFrame};
+
 mod framed;
 pub use self::framed::{Framed, FramedParts};
 

tokio-util/src/compat.rs

@@ -0,0 +1,201 @@
+//! Compatibility between the `tokio::io` and `futures-io` versions of the
+//! `AsyncRead` and `AsyncWrite` traits.
+use pin_project_lite::pin_project;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+pin_project! {
+    /// A compatibility layer that allows conversion between the
+    /// `tokio::io` and `futures-io` `AsyncRead` and `AsyncWrite` traits.
+    #[derive(Copy, Clone, Debug)]
+    pub struct Compat<T> {
+        #[pin]
+        inner: T,
+    }
+}
+
+/// Extension trait that allows converting a type implementing
+/// `futures_io::AsyncRead` to implement `tokio::io::AsyncRead`.
+pub trait FuturesAsyncReadCompatExt: futures_io::AsyncRead {
+    /// Wraps `self` with a compatibility layer that implements
+    /// `tokio_io::AsyncWrite`.
+    fn compat(self) -> Compat<Self>
+    where
+        Self: Sized,
+    {
+        Compat::new(self)
+    }
+}
+
+impl<T: futures_io::AsyncRead> FuturesAsyncReadCompatExt for T {}
+
+/// Extension trait that allows converting a type implementing
+/// `futures_io::AsyncWrite` to implement `tokio::io::AsyncWrite`.
+pub trait FuturesAsyncWriteCompatExt: futures_io::AsyncWrite {
+    /// Wraps `self` with a compatibility layer that implements
+    /// `tokio::io::AsyncWrite`.
+    fn compat_write(self) -> Compat<Self>
+    where
+        Self: Sized,
+    {
+        Compat::new(self)
+    }
+}
+
+impl<T: futures_io::AsyncWrite> FuturesAsyncWriteCompatExt for T {}
+
+/// Extension trait that allows converting a type implementing
+/// `tokio::io::AsyncRead` to implement `futures_io::AsyncRead`.
+pub trait Tokio02AsyncReadCompatExt: tokio::io::AsyncRead {
+    /// Wraps `self` with a compatibility layer that implements
+    /// `futures_io::AsyncRead`.
+    fn compat(self) -> Compat<Self>
+    where
+        Self: Sized,
+    {
+        Compat::new(self)
+    }
+}
+
+impl<T: tokio::io::AsyncRead> Tokio02AsyncReadCompatExt for T {}
+
+/// Extension trait that allows converting a type implementing
+/// `tokio::io::AsyncWrite` to implement `futures_io::AsyncWrite`.
+pub trait Tokio02AsyncWriteCompatExt: tokio::io::AsyncWrite {
+    /// Wraps `self` with a compatibility layer that implements
+    /// `futures_io::AsyncWrite`.
+    fn compat_write(self) -> Compat<Self>
+    where
+        Self: Sized,
+    {
+        Compat::new(self)
+    }
+}
+
+impl<T: tokio::io::AsyncWrite> Tokio02AsyncWriteCompatExt for T {}
+
+// === impl Compat ===
+
+impl<T> Compat<T> {
+    fn new(inner: T) -> Self {
+        Self { inner }
+    }
+
+    /// Get a reference to the `Future`, `Stream`, `AsyncRead`, or `AsyncWrite` object
+    /// contained within.
+    pub fn get_ref(&self) -> &T {
+        &self.inner
+    }
+
+    /// Get a mutable reference to the `Future`, `Stream`, `AsyncRead`, or `AsyncWrite` object
+    /// contained within.
+    pub fn get_mut(&mut self) -> &mut T {
+        &mut self.inner
+    }
+
+    /// Returns the wrapped item.
+    pub fn into_inner(self) -> T {
+        self.inner
+    }
+}
+
+impl<T> tokio::io::AsyncRead for Compat<T>
+where
+    T: futures_io::AsyncRead,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<io::Result<usize>> {
+        futures_io::AsyncRead::poll_read(self.project().inner, cx, buf)
+    }
+}
+
+impl<T> futures_io::AsyncRead for Compat<T>
+where
+    T: tokio::io::AsyncRead,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<io::Result<usize>> {
+        tokio::io::AsyncRead::poll_read(self.project().inner, cx, buf)
+    }
+}
+
+impl<T> tokio::io::AsyncBufRead for Compat<T>
+where
+    T: futures_io::AsyncBufRead,
+{
+    fn poll_fill_buf<'a>(
+        self: Pin<&'a mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<io::Result<&'a [u8]>> {
+        futures_io::AsyncBufRead::poll_fill_buf(self.project().inner, cx)
+    }
+
+    fn consume(self: Pin<&mut Self>, amt: usize) {
+        futures_io::AsyncBufRead::consume(self.project().inner, amt)
+    }
+}
+
+impl<T> futures_io::AsyncBufRead for Compat<T>
+where
+    T: tokio::io::AsyncBufRead,
+{
+    fn poll_fill_buf<'a>(
+        self: Pin<&'a mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<io::Result<&'a [u8]>> {
+        tokio::io::AsyncBufRead::poll_fill_buf(self.project().inner, cx)
+    }
+
+    fn consume(self: Pin<&mut Self>, amt: usize) {
+        tokio::io::AsyncBufRead::consume(self.project().inner, amt)
+    }
+}
+
+impl<T> tokio::io::AsyncWrite for Compat<T>
+where
+    T: futures_io::AsyncWrite,
+{
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        futures_io::AsyncWrite::poll_write(self.project().inner, cx, buf)
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        futures_io::AsyncWrite::poll_flush(self.project().inner, cx)
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        futures_io::AsyncWrite::poll_close(self.project().inner, cx)
+    }
+}
+
+impl<T> futures_io::AsyncWrite for Compat<T>
+where
+    T: tokio::io::AsyncWrite,
+{
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        tokio::io::AsyncWrite::poll_write(self.project().inner, cx, buf)
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        tokio::io::AsyncWrite::poll_flush(self.project().inner, cx)
+    }
+
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        tokio::io::AsyncWrite::poll_shutdown(self.project().inner, cx)
+    }
+}

tokio-util/src/lib.rs

@@ -1,11 +1,12 @@
-#![doc(html_root_url = "https://docs.rs/tokio-util/0.2.0")]
+#![doc(html_root_url = "https://docs.rs/tokio-util/0.3.1")]
+#![allow(clippy::needless_doctest_main)]
 #![warn(
     missing_debug_implementations,
     missing_docs,
     rust_2018_idioms,
     unreachable_pub
 )]
-#![deny(intra_doc_link_resolution_failure)]
+#![cfg_attr(docsrs, deny(broken_intra_doc_links))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
@@ -13,6 +14,12 @@
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
 //! Utilities for working with Tokio.
+//!
+//! This crate is not versioned in lockstep with the core
+//! [`tokio`] crate. However, `tokio-util` _will_ respect Rust's
+//! semantic versioning policy, especially with regard to breaking changes.
+//!
+//! [`tokio`]: https://docs.rs/tokio
 
 #[macro_use]
 mod cfg;
@@ -24,3 +31,7 @@ cfg_codec! {
 cfg_udp! {
     pub mod udp;
 }
+
+cfg_compat! {
+    pub mod compat;
+}

tokio-util/src/udp/frame.rs

@@ -6,6 +6,7 @@ use bytes::{BufMut, BytesMut};
 use futures_core::ready;
 use futures_sink::Sink;
 use std::io;
+use std::mem::MaybeUninit;
 use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -27,7 +28,7 @@ use std::task::{Context, Poll};
 /// calling `split` on the `UdpFramed` returned by this method, which will break
 /// them into separate objects, allowing them to interact more easily.
 #[must_use = "sinks do nothing unless polled"]
-#[cfg_attr(docsrs, doc(feature = "codec-udp"))]
+#[cfg_attr(docsrs, doc(all(feature = "codec", feature = "udp")))]
 #[derive(Debug)]
 pub struct UdpFramed<C> {
     socket: UdpSocket,
@@ -36,6 +37,8 @@ pub struct UdpFramed<C> {
     wr: BytesMut,
     out_addr: SocketAddr,
     flushed: bool,
+    is_readable: bool,
+    current_addr: Option<SocketAddr>,
 }
 
 impl<C: Decoder + Unpin> Stream for UdpFramed<C> {
@@ -46,31 +49,43 @@ impl<C: Decoder + Unpin> Stream for UdpFramed<C> {
 
         pin.rd.reserve(INITIAL_RD_CAPACITY);
 
-        let (_n, addr) = unsafe {
-            // Read into the buffer without having to initialize the memory.
-            //
-            // safety: we know tokio::net::UdpSocket never reads from the memory
-            // during a recv
-            let res = {
-                let bytes = &mut *(pin.rd.bytes_mut() as *mut _ as *mut [u8]);
-                ready!(Pin::new(&mut pin.socket).poll_recv_from(cx, bytes))
+        loop {
+            // Are there are still bytes left in the read buffer to decode?
+            if pin.is_readable {
+                if let Some(frame) = pin.codec.decode_eof(&mut pin.rd)? {
+                    let current_addr = pin
+                        .current_addr
+                        .expect("will always be set before this line is called");
+
+                    return Poll::Ready(Some(Ok((frame, current_addr))));
+                }
+
+                // if this line has been reached then decode has returned `None`.
+                pin.is_readable = false;
+                pin.rd.clear();
+            }
+
+            // We're out of data. Try and fetch more data to decode
+            let addr = unsafe {
+                // Convert `&mut [MaybeUnit<u8>]` to `&mut [u8]` because we will be
+                // writing to it via `poll_recv_from` and therefore initializing the memory.
+                let buf: &mut [u8] =
+                    &mut *(pin.rd.bytes_mut() as *mut [MaybeUninit<u8>] as *mut [u8]);
+
+                let res = ready!(Pin::new(&mut pin.socket).poll_recv_from(cx, buf));
+
+                let (n, addr) = res?;
+                pin.rd.advance_mut(n);
+                addr
             };
 
-            let (n, addr) = res?;
-            pin.rd.advance_mut(n);
-            (n, addr)
-        };
-
-        let frame_res = pin.codec.decode(&mut pin.rd);
-        pin.rd.clear();
-        let frame = frame_res?;
-        let result = frame.map(|frame| Ok((frame, addr))); // frame -> (frame, addr)
-
-        Poll::Ready(result)
+            pin.current_addr = Some(addr);
+            pin.is_readable = true;
+        }
     }
 }
 
-impl<C: Encoder + Unpin> Sink<(C::Item, SocketAddr)> for UdpFramed<C> {
+impl<I, C: Encoder<I> + Unpin> Sink<(I, SocketAddr)> for UdpFramed<C> {
     type Error = C::Error;
 
     fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
@@ -84,7 +99,7 @@ impl<C: Encoder + Unpin> Sink<(C::Item, SocketAddr)> for UdpFramed<C> {
         Poll::Ready(Ok(()))
     }
 
-    fn start_send(self: Pin<&mut Self>, item: (C::Item, SocketAddr)) -> Result<(), Self::Error> {
+    fn start_send(self: Pin<&mut Self>, item: (I, SocketAddr)) -> Result<(), Self::Error> {
         let (frame, out_addr) = item;
 
         let pin = self.get_mut();
@@ -148,6 +163,8 @@ impl<C> UdpFramed<C> {
             rd: BytesMut::with_capacity(INITIAL_RD_CAPACITY),
             wr: BytesMut::with_capacity(INITIAL_WR_CAPACITY),
             flushed: true,
+            is_readable: false,
+            current_addr: None,
         }
     }
 

tokio-util/tests/codecs.rs

@@ -209,9 +209,9 @@ fn lines_encoder() {
     let mut codec = LinesCodec::new();
     let mut buf = BytesMut::new();
 
-    codec.encode(String::from("line 1"), &mut buf).unwrap();
+    codec.encode("line 1", &mut buf).unwrap();
     assert_eq!("line 1\n", buf);
 
-    codec.encode(String::from("line 2"), &mut buf).unwrap();
+    codec.encode("line 2", &mut buf).unwrap();
     assert_eq!("line 1\nline 2\n", buf);
 }

tokio-util/tests/framed.rs

@@ -28,8 +28,7 @@ impl Decoder for U32Codec {
     }
 }
 
-impl Encoder for U32Codec {
-    type Item = u32;
+impl Encoder<u32> for U32Codec {
     type Error = io::Error;
 
     fn encode(&mut self, item: u32, dst: &mut BytesMut) -> io::Result<()> {

tokio-util/tests/framed_write.rs

@@ -28,8 +28,7 @@ macro_rules! pin {
 
 struct U32Encoder;
 
-impl Encoder for U32Encoder {
-    type Item = u32;
+impl Encoder<u32> for U32Encoder {
     type Error = io::Error;
 
     fn encode(&mut self, item: u32, dst: &mut BytesMut) -> io::Result<()> {