fs: add support for non-threadpool executors (#1495)

Provides a thread pool dedicated to running blocking operations (#588)
and update `tokio-fs` to use this pool.

In an effort to make incremental progress, this is an initial step
towards a final solution. First, it provides a very basic pool
implementation with the intend that the pool will be
replaced before the final release. Second, it updates `tokio-fs` to
always use this blocking pool instead of conditionally using
`threadpool::blocking`. Issue #588 contains additional discussion around
potential improvements to the "blocking for all" strategy.

The implementation provided here builds on work started in #954 and
continued in #1045. The general idea is th same as #1045, but the PR
improves on some of the details:

* The number of explicit operations tracked by `File` is reduced only to
  the ones that could interact. All other ops are spawned on the
  blocking pool without being tracked by the `File` instance.

* The `seek` implementation is not backed by a trait and `poll_seek`
  function. This avoids the question of how to model non-blocking seeks
  on top of a blocking file. In this patch, `seek` is represented as an
  `async fn`. If the associated future is dropped before the caller
  observes the return value, we make no effort to define the state in
  which the file ends up.

tokio-fs/tests/sys/file.rs

@@ -0,0 +1,265 @@
+use std::collections::VecDeque;
+use std::fmt;
+use std::fs::{Metadata, Permissions};
+use std::io;
+use std::io::prelude::*;
+use std::io::SeekFrom;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+
+pub struct File {
+    shared: Arc<Mutex<Shared>>,
+}
+
+pub struct Handle {
+    shared: Arc<Mutex<Shared>>,
+}
+
+struct Shared {
+    calls: VecDeque<Call>,
+}
+
+#[derive(Debug)]
+enum Call {
+    Read(io::Result<Vec<u8>>),
+    Write(io::Result<Vec<u8>>),
+    Seek(SeekFrom, io::Result<u64>),
+    SyncAll(io::Result<()>),
+    SyncData(io::Result<()>),
+    SetLen(u64, io::Result<()>),
+}
+
+impl Handle {
+    pub fn read(&self, data: &[u8]) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls.push_back(Call::Read(Ok(data.to_owned())));
+        self
+    }
+
+    pub fn read_err(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::Read(Err(io::ErrorKind::Other.into())));
+        self
+    }
+
+    pub fn write(&self, data: &[u8]) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls.push_back(Call::Write(Ok(data.to_owned())));
+        self
+    }
+
+    pub fn write_err(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::Write(Err(io::ErrorKind::Other.into())));
+        self
+    }
+
+    pub fn seek_start_ok(&self, offset: u64) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::Seek(SeekFrom::Start(offset), Ok(offset)));
+        self
+    }
+
+    pub fn seek_current_ok(&self, offset: i64, ret: u64) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::Seek(SeekFrom::Current(offset), Ok(ret)));
+        self
+    }
+
+    pub fn sync_all(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls.push_back(Call::SyncAll(Ok(())));
+        self
+    }
+
+    pub fn sync_all_err(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::SyncAll(Err(io::ErrorKind::Other.into())));
+        self
+    }
+
+    pub fn sync_data(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls.push_back(Call::SyncData(Ok(())));
+        self
+    }
+
+    pub fn sync_data_err(&self) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::SyncData(Err(io::ErrorKind::Other.into())));
+        self
+    }
+
+    pub fn set_len(&self, size: u64) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls.push_back(Call::SetLen(size, Ok(())));
+        self
+    }
+
+    pub fn set_len_err(&self, size: u64) -> &Self {
+        let mut s = self.shared.lock().unwrap();
+        s.calls
+            .push_back(Call::SetLen(size, Err(io::ErrorKind::Other.into())));
+        self
+    }
+
+    pub fn remaining(&self) -> usize {
+        let s = self.shared.lock().unwrap();
+        s.calls.len()
+    }
+}
+
+impl Drop for Handle {
+    fn drop(&mut self) {
+        if !std::thread::panicking() {
+            let s = self.shared.lock().unwrap();
+            assert_eq!(0, s.calls.len());
+        }
+    }
+}
+
+impl File {
+    pub fn open(_: PathBuf) -> io::Result<File> {
+        unimplemented!();
+    }
+
+    pub fn create(_: PathBuf) -> io::Result<File> {
+        unimplemented!();
+    }
+
+    pub fn mock() -> (Handle, File) {
+        let shared = Arc::new(Mutex::new(Shared {
+            calls: VecDeque::new(),
+        }));
+
+        let handle = Handle {
+            shared: shared.clone(),
+        };
+        let file = File { shared };
+
+        (handle, file)
+    }
+
+    pub fn sync_all(&self) -> io::Result<()> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(SyncAll(ret)) => ret,
+            Some(op) => panic!("expected next call to be {:?}; was sync_all", op),
+            None => panic!("did not expect call"),
+        }
+    }
+
+    pub fn sync_data(&self) -> io::Result<()> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(SyncData(ret)) => ret,
+            Some(op) => panic!("expected next call to be {:?}; was sync_all", op),
+            None => panic!("did not expect call"),
+        }
+    }
+
+    pub fn set_len(&self, size: u64) -> io::Result<()> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(SetLen(arg, ret)) => {
+                assert_eq!(arg, size);
+                ret
+            }
+            Some(op) => panic!("expected next call to be {:?}; was sync_all", op),
+            None => panic!("did not expect call"),
+        }
+    }
+
+    pub fn metadata(&self) -> io::Result<Metadata> {
+        unimplemented!();
+    }
+
+    pub fn set_permissions(&self, _perm: Permissions) -> io::Result<()> {
+        unimplemented!();
+    }
+
+    pub fn try_clone(&self) -> io::Result<Self> {
+        unimplemented!();
+    }
+}
+
+impl Read for &'_ File {
+    fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(Read(Ok(data))) => {
+                assert!(dst.len() >= data.len());
+                assert!(dst.len() <= 16 * 1024, "actual = {}", dst.len()); // max buffer
+
+                &mut dst[..data.len()].copy_from_slice(&data);
+                Ok(data.len())
+            }
+            Some(Read(Err(e))) => Err(e),
+            Some(op) => panic!("expected next call to be {:?}; was a read", op),
+            None => panic!("did not expect call"),
+        }
+    }
+}
+
+impl Write for &'_ File {
+    fn write(&mut self, src: &[u8]) -> io::Result<usize> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(Write(Ok(data))) => {
+                assert_eq!(src, &data[..]);
+                Ok(src.len())
+            }
+            Some(Write(Err(e))) => Err(e),
+            Some(op) => panic!("expected next call to be {:?}; was write", op),
+            None => panic!("did not expect call"),
+        }
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+impl Seek for &'_ File {
+    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+        use self::Call::*;
+
+        let mut s = self.shared.lock().unwrap();
+
+        match s.calls.pop_front() {
+            Some(Seek(expect, res)) => {
+                assert_eq!(expect, pos);
+                res
+            }
+            Some(op) => panic!("expected call {:?}; was `seek`", op),
+            None => panic!("did not expect call; was `seek`"),
+        }
+    }
+}
+
+impl fmt::Debug for File {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt.debug_struct("mock::File").finish()
+    }
+}

tokio-fs/tests/sys/pool.rs

@@ -0,0 +1,66 @@
+use tokio_sync::oneshot;
+
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use std::future::Future;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+thread_local! {
+    static QUEUE: RefCell<VecDeque<Box<dyn FnOnce() + Send>>> = RefCell::new(VecDeque::new())
+}
+
+#[derive(Debug)]
+pub(crate) struct Blocking<T> {
+    rx: oneshot::Receiver<T>,
+}
+
+pub(crate) fn run<F, R>(f: F) -> Blocking<R>
+where
+    F: FnOnce() -> R + Send + 'static,
+    R: Send + 'static,
+{
+    let (tx, rx) = oneshot::channel();
+    let task = Box::new(move || {
+        let _ = tx.send(f());
+    });
+
+    QUEUE.with(|cell| cell.borrow_mut().push_back(task));
+
+    Blocking { rx }
+}
+
+impl<T> Future for Blocking<T> {
+    type Output = T;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        use std::task::Poll::*;
+
+        match Pin::new(&mut self.rx).poll(cx) {
+            Ready(Ok(v)) => Ready(v),
+            Ready(Err(e)) => panic!("error = {:?}", e),
+            Pending => Pending,
+        }
+    }
+}
+
+pub(crate) async fn asyncify<F, T>(f: F) -> io::Result<T>
+where
+    F: FnOnce() -> io::Result<T> + Send + 'static,
+    T: Send + 'static,
+{
+    run(f).await
+}
+
+pub(crate) fn len() -> usize {
+    QUEUE.with(|cell| cell.borrow().len())
+}
+
+pub(crate) fn run_one() {
+    let task = QUEUE
+        .with(|cell| cell.borrow_mut().pop_front())
+        .expect("expected task to run, but none ready");
+
+    task();
+}