mirror of
https://github.com/tokio-rs/tokio.git
synced 2026-08-22 00:00:11 +02:00
Carl Lerche
08e20fcf6a
Provides a thread pool dedicated to running blocking operations (#588) and update `tokio-fs` to use this pool. In an effort to make incremental progress, this is an initial step towards a final solution. First, it provides a very basic pool implementation with the intend that the pool will be replaced before the final release. Second, it updates `tokio-fs` to always use this blocking pool instead of conditionally using `threadpool::blocking`. Issue #588 contains additional discussion around potential improvements to the "blocking for all" strategy. The implementation provided here builds on work started in #954 and continued in #1045. The general idea is th same as #1045, but the PR improves on some of the details: * The number of explicit operations tracked by `File` is reduced only to the ones that could interact. All other ops are spawned on the blocking pool without being tracked by the `File` instance. * The `seek` implementation is not backed by a trait and `poll_seek` function. This avoids the question of how to model non-blocking seeks on top of a blocking file. In this patch, `seek` is represented as an `async fn`. If the associated future is dropped before the caller observes the return value, we make no effort to define the state in which the file ends up.
737 lines
15 KiB
Rust
737 lines
15 KiB
Rust
#![warn(rust_2018_idioms)]
|
|
|
|
mod sys {
|
|
mod file;
|
|
pub(crate) mod pool;
|
|
|
|
pub(crate) use self::file::File;
|
|
pub(crate) use self::pool::{run, Blocking};
|
|
}
|
|
use sys::pool::{self, asyncify};
|
|
|
|
#[allow(warnings)]
|
|
#[path = "../src/file.rs"]
|
|
mod file;
|
|
use file::File;
|
|
|
|
#[allow(warnings)]
|
|
#[path = "../src/blocking.rs"]
|
|
mod blocking;
|
|
|
|
use tokio::prelude::*;
|
|
use tokio_test::{assert_pending, assert_ready, assert_ready_err, assert_ready_ok, task};
|
|
|
|
use std::io::SeekFrom;
|
|
|
|
const HELLO: &[u8] = b"hello world...";
|
|
const FOO: &[u8] = b"foo bar baz...";
|
|
|
|
#[test]
|
|
fn open_read() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 1024];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_eq!(0, pool::len());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, mock.remaining());
|
|
assert_eq!(1, pool::len());
|
|
|
|
pool::run_one();
|
|
|
|
assert_eq!(0, mock.remaining());
|
|
assert!(t.is_woken());
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, HELLO.len());
|
|
assert_eq!(&buf[..n], HELLO);
|
|
}
|
|
|
|
#[test]
|
|
fn read_twice_before_dispatch() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 1024];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_pending!(t.poll());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(pool::len(), 1);
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(&buf[..n], HELLO);
|
|
}
|
|
|
|
#[test]
|
|
fn read_with_smaller_buf() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
{
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
{
|
|
let mut buf = [0; 4];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, 4);
|
|
assert_eq!(&buf[..], &HELLO[..n]);
|
|
}
|
|
|
|
// Calling again immediately succeeds with the rest of the buffer
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, 10);
|
|
assert_eq!(&buf[..n], &HELLO[4..]);
|
|
|
|
assert_eq!(0, pool::len());
|
|
}
|
|
|
|
#[test]
|
|
fn read_with_bigger_buf() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(&HELLO[..4]).read(&HELLO[4..]);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
{
|
|
let mut buf = [0; 4];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
{
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, 4);
|
|
assert_eq!(&buf[..n], &HELLO[..n]);
|
|
}
|
|
|
|
// Calling again immediately succeeds with the rest of the buffer
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, 10);
|
|
assert_eq!(&buf[..n], &HELLO[4..]);
|
|
|
|
assert_eq!(0, pool::len());
|
|
}
|
|
|
|
#[test]
|
|
fn read_err_then_read_success() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read_err().read(&HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
{
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
{
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
|
|
assert_eq!(n, HELLO.len());
|
|
assert_eq!(&buf[..n], HELLO);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn open_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
|
|
assert_eq!(0, pool::len());
|
|
assert_ready_ok!(t.poll());
|
|
|
|
assert_eq!(1, mock.remaining());
|
|
assert_eq!(1, pool::len());
|
|
|
|
pool::run_one();
|
|
|
|
assert_eq!(0, mock.remaining());
|
|
assert!(!t.is_woken());
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn flush_while_idle() {
|
|
let (_mock, file) = sys::File::mock();
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn read_with_buffer_larger_than_max() {
|
|
// Chunks
|
|
let a = 16 * 1024;
|
|
let b = a * 2;
|
|
let c = a * 3;
|
|
let d = a * 4;
|
|
|
|
assert_eq!(d / 1024, 64);
|
|
|
|
let mut data = vec![];
|
|
for i in 0..(d - 1) {
|
|
data.push((i % 151) as u8);
|
|
}
|
|
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(&data[0..a])
|
|
.read(&data[a..b])
|
|
.read(&data[b..c])
|
|
.read(&data[c..]);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut actual = vec![0; d];
|
|
let mut pos = 0;
|
|
|
|
while pos < data.len() {
|
|
let mut t = task::spawn(file.read(&mut actual[pos..]));
|
|
|
|
assert_pending!(t.poll());
|
|
pool::run_one();
|
|
assert!(t.is_woken());
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert!(n <= a);
|
|
|
|
pos += n;
|
|
}
|
|
|
|
assert_eq!(mock.remaining(), 0);
|
|
assert_eq!(data, &actual[..data.len()]);
|
|
}
|
|
|
|
#[test]
|
|
fn write_with_buffer_larger_than_max() {
|
|
// Chunks
|
|
let a = 16 * 1024;
|
|
let b = a * 2;
|
|
let c = a * 3;
|
|
let d = a * 4;
|
|
|
|
assert_eq!(d / 1024, 64);
|
|
|
|
let mut data = vec![];
|
|
for i in 0..(d - 1) {
|
|
data.push((i % 151) as u8);
|
|
}
|
|
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(&data[0..a])
|
|
.write(&data[a..b])
|
|
.write(&data[b..c])
|
|
.write(&data[c..]);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut rem = &data[..];
|
|
|
|
let mut first = true;
|
|
|
|
while !rem.is_empty() {
|
|
let mut t = task::spawn(file.write(rem));
|
|
|
|
if !first {
|
|
assert_pending!(t.poll());
|
|
pool::run_one();
|
|
assert!(t.is_woken());
|
|
}
|
|
|
|
first = false;
|
|
|
|
let n = assert_ready_ok!(t.poll());
|
|
|
|
rem = &rem[n..];
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
assert_eq!(mock.remaining(), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn write_twice_before_dispatch() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).write(FOO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(pool::len(), 1);
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(pool::len(), 1);
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn incomplete_read_followed_by_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO)
|
|
.seek_current_ok(-(HELLO.len() as i64), 0)
|
|
.write(FOO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 32];
|
|
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
assert_eq!(pool::len(), 1);
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn incomplete_partial_read_followed_by_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO).seek_current_ok(-10, 0).write(FOO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 32];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut buf = [0; 4];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
assert_eq!(pool::len(), 1);
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn incomplete_read_followed_by_flush() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO)
|
|
.seek_current_ok(-(HELLO.len() as i64), 0)
|
|
.write(FOO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 32];
|
|
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
}
|
|
|
|
#[test]
|
|
fn incomplete_flush_followed_by_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).write(FOO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
let n = assert_ready_ok!(t.poll());
|
|
assert_eq!(n, HELLO.len());
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_pending!(t.poll());
|
|
|
|
// TODO: Move under write
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn read_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read_err();
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut buf = [0; 1024];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
assert!(t.is_woken());
|
|
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn write_write_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write_err();
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn write_read_write_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write_err().read(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut buf = [0; 1024];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn write_read_flush_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write_err().read(HELLO);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut buf = [0; 1024];
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
|
|
assert_pending!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn write_seek_write_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write_err().seek_start_ok(0);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
{
|
|
let mut t = task::spawn(file.seek(SeekFrom::Start(0)));
|
|
assert_pending!(t.poll());
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.write(FOO));
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn write_seek_flush_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write_err().seek_start_ok(0);
|
|
|
|
let mut file = File::from_std(file);
|
|
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
pool::run_one();
|
|
|
|
{
|
|
let mut t = task::spawn(file.seek(SeekFrom::Start(0)));
|
|
assert_pending!(t.poll());
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
let mut t = task::spawn(file.flush());
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn sync_all_ordered_after_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).sync_all();
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.sync_all());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn sync_all_err_ordered_after_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).sync_all_err();
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.sync_all());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn sync_data_ordered_after_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).sync_data();
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.sync_data());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn sync_data_err_ordered_after_write() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.write(HELLO).sync_data_err();
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.write(HELLO));
|
|
assert_ready_ok!(t.poll());
|
|
|
|
let mut t = task::spawn(file.sync_data());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_pending!(t.poll());
|
|
|
|
assert_eq!(1, pool::len());
|
|
pool::run_one();
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn open_set_len_ok() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.set_len(123);
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.set_len(123));
|
|
|
|
assert_pending!(t.poll());
|
|
assert_eq!(1, mock.remaining());
|
|
|
|
pool::run_one();
|
|
assert_eq!(0, mock.remaining());
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn open_set_len_err() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.set_len_err(123);
|
|
|
|
let mut file = File::from_std(file);
|
|
let mut t = task::spawn(file.set_len(123));
|
|
|
|
assert_pending!(t.poll());
|
|
assert_eq!(1, mock.remaining());
|
|
|
|
pool::run_one();
|
|
assert_eq!(0, mock.remaining());
|
|
|
|
assert!(t.is_woken());
|
|
assert_ready_err!(t.poll());
|
|
}
|
|
|
|
#[test]
|
|
fn partial_read_set_len_ok() {
|
|
let (mock, file) = sys::File::mock();
|
|
mock.read(HELLO)
|
|
.seek_current_ok(-14, 0)
|
|
.set_len(123)
|
|
.read(FOO);
|
|
|
|
let mut buf = [0; 32];
|
|
let mut file = File::from_std(file);
|
|
|
|
{
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
}
|
|
|
|
pool::run_one();
|
|
|
|
{
|
|
let mut t = task::spawn(file.set_len(123));
|
|
|
|
assert_pending!(t.poll());
|
|
pool::run_one();
|
|
assert_ready_ok!(t.poll());
|
|
}
|
|
|
|
let mut t = task::spawn(file.read(&mut buf));
|
|
assert_pending!(t.poll());
|
|
pool::run_one();
|
|
let n = assert_ready_ok!(t.poll());
|
|
|
|
assert_eq!(n, FOO.len());
|
|
assert_eq!(&buf[..n], FOO);
|
|
}
|