tokio-rs/tokio
1
0
Fork 0
mirror of https://github.com/tokio-rs/tokio.git synced 2026-08-27 00:00:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

fs: add support for non-threadpool executors (#1495)

Browse Source 
Provides a thread pool dedicated to running blocking operations (#588)
and update `tokio-fs` to use this pool.

In an effort to make incremental progress, this is an initial step
towards a final solution. First, it provides a very basic pool
implementation with the intend that the pool will be
replaced before the final release. Second, it updates `tokio-fs` to
always use this blocking pool instead of conditionally using
`threadpool::blocking`. Issue #588 contains additional discussion around
potential improvements to the "blocking for all" strategy.

The implementation provided here builds on work started in #954 and
continued in #1045. The general idea is th same as #1045, but the PR
improves on some of the details:

* The number of explicit operations tracked by `File` is reduced only to
  the ones that could interact. All other ops are spawned on the
  blocking pool without being tracked by the `File` instance.

* The `seek` implementation is not backed by a trait and `poll_seek`
  function. This avoids the question of how to model non-blocking seeks
  on top of a blocking file. In this patch, `seek` is represented as an
  `async fn`. If the associated future is dropped before the caller
  observes the return value, we make no effort to define the state in
  which the file ends up.
This commit is contained in:
Carl Lerche
2019-08-27 12:25:20 -07:00
committed by GitHub
parent 08099bb2d3
commit 08e20fcf6a
35 changed files with 1974 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tokio-fs/src/blocking.rs
+252
View File
@@ -0,0 +1,252 @@
use crate::sys;
use tokio_io::{AsyncRead, AsyncWrite};
use futures_core::ready;
use std::cmp;
use std::future::Future;
use std::io;
use std::io::prelude::*;
use std::pin::Pin;
use std::task::Poll::*;
use std::task::{Context, Poll};
use self::State::*;
/// `T` should not implement _both_ Read and Write.
#[derive(Debug)]
pub(crate) struct Blocking<T> {
inner: Option<T>,
state: State<T>,
}
#[derive(Debug)]
pub(crate) struct Buf {
buf: Vec<u8>,
pos: usize,
}
pub(crate) const MAX_BUF: usize = 16 * 1024;
#[derive(Debug)]
enum State<T> {
Idle(Option<Buf>),
Busy(sys::Blocking<(io::Result<usize>, Buf, T)>),
}
impl<T> Blocking<T> {
pub(crate) fn new(inner: T) -> Blocking<T> {
Blocking {
inner: Some(inner),
state: State::Idle(Some(Buf::with_capacity(0))),
}
}
}
impl<T> AsyncRead for Blocking<T>
where
T: Read + Unpin + Send + 'static,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
dst: &mut [u8],
) -> Poll<io::Result<usize>> {
loop {
match self.state {
Idle(ref mut buf_cell) => {
let mut buf = buf_cell.take().unwrap();
if !buf.is_empty() {
let n = buf.copy_to(dst);
*buf_cell = Some(buf);
return Ready(Ok(n));
}
buf.ensure_capacity_for(dst);
let mut inner = self.inner.take().unwrap();
self.state = Busy(sys::run(move || {
let res = buf.read_from(&mut inner);
(res, buf, inner)
}));
}
Busy(ref mut rx) => {
let (res, mut buf, inner) = ready!(Pin::new(rx).poll(cx));
self.inner = Some(inner);
match res {
Ok(_) => {
let n = buf.copy_to(dst);
self.state = Idle(Some(buf));
return Ready(Ok(n));
}
Err(e) => {
assert!(buf.is_empty());
self.state = Idle(Some(buf));
return Ready(Err(e));
}
}
}
}
}
}
}
impl<T> AsyncWrite for Blocking<T>
where
T: Write + Unpin + Send + 'static,
{
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
src: &[u8],
) -> Poll<io::Result<usize>> {
loop {
match self.state {
Idle(ref mut buf_cell) => {
let mut buf = buf_cell.take().unwrap();
assert!(buf.is_empty());
let n = buf.copy_from(src);
let mut inner = self.inner.take().unwrap();
self.state = Busy(sys::run(move || {
let n = buf.len();
let res = buf.write_to(&mut inner).map(|_| n);
(res, buf, inner)
}));
return Ready(Ok(n));
}
Busy(ref mut rx) => {
let (res, buf, inner) = ready!(Pin::new(rx).poll(cx));
self.state = Idle(Some(buf));
self.inner = Some(inner);
// If error, return
res?;
}
}
}
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
let (res, buf, inner) = match self.state {
Idle(_) => return Ready(Ok(())),
Busy(ref mut rx) => ready!(Pin::new(rx).poll(cx)),
};
// The buffer is not used here
self.state = Idle(Some(buf));
self.inner = Some(inner);
Ready(res.map(|_| ()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
}
}
/// Repeates operations that are interrupted
macro_rules! uninterruptibly {
($e:expr) => {{
loop {
match $e {
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
res => break res,
}
}
}};
}
impl Buf {
pub(crate) fn with_capacity(n: usize) -> Buf {
Buf {
buf: Vec::with_capacity(n),
pos: 0,
}
}
pub(crate) fn is_empty(&self) -> bool {
self.len() == 0
}
pub(crate) fn len(&self) -> usize {
self.buf.len() - self.pos
}
pub(crate) fn copy_to(&mut self, dst: &mut [u8]) -> usize {
let n = cmp::min(self.len(), dst.len());
dst[..n].copy_from_slice(&self.bytes()[..n]);
self.pos += n;
if self.pos == self.buf.len() {
self.buf.truncate(0);
self.pos = 0;
}
n
}
pub(crate) fn copy_from(&mut self, src: &[u8]) -> usize {
assert!(self.is_empty());
let n = cmp::min(src.len(), MAX_BUF);
self.buf.extend_from_slice(&src[..n]);
n
}
pub(crate) fn bytes(&self) -> &[u8] {
&self.buf[self.pos..]
}
pub(crate) fn ensure_capacity_for(&mut self, bytes: &[u8]) {
assert!(self.is_empty());
let len = cmp::min(bytes.len(), MAX_BUF);
if self.buf.len() < len {
self.buf.reserve(len - self.buf.len());
}
unsafe {
self.buf.set_len(len);
}
}
pub(crate) fn read_from<T: Read>(&mut self, rd: &mut T) -> io::Result<usize> {
let res = uninterruptibly!(rd.read(&mut self.buf));
if let Ok(n) = res {
self.buf.truncate(n);
} else {
self.buf.clear();
}
assert_eq!(self.pos, 0);
res
}
pub(crate) fn write_to<T: Write>(&mut self, wr: &mut T) -> io::Result<()> {
assert_eq!(self.pos, 0);
// `write_all` already ignores interrupts
let res = wr.write_all(&self.buf);
self.buf.clear();
res
}
pub(crate) fn discard_read(&mut self) -> i64 {
let ret = -(self.bytes().len() as i64);
self.pos = 0;
self.buf.truncate(0);
ret
}
}
tokio-fs/src/create_dir.rs
+2 -1
View File
@@ -9,5 +9,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.create_dir.html
pub async fn create_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
asyncify(|| std::fs::create_dir(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::create_dir(path)).await
}
tokio-fs/src/create_dir_all.rs
+2 -1
View File
@@ -10,5 +10,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.create_dir_all.html
pub async fn create_dir_all<P: AsRef<Path>>(path: P) -> io::Result<()> {
asyncify(|| std::fs::create_dir_all(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::create_dir_all(path)).await
}
tokio-fs/src/file.rs
+275 -59
View File
@@ -2,17 +2,23 @@
//!
//! [`File`]: file/struct.File.html
use crate::{asyncify, blocking_io, OpenOptions};
use self::State::*;
use crate::blocking::Buf;
use crate::{asyncify, sys};
use tokio_io::{AsyncRead, AsyncWrite};
use std::convert::TryFrom;
use futures_core::ready;
use std::fmt;
use std::fs::{Metadata, Permissions};
use std::io::{self, Read, Seek, Write};
use std::future::Future;
use std::io::{self, Seek, SeekFrom};
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
use std::task::Poll::*;
/// A reference to an open file on the filesystem.
///
@@ -58,9 +64,27 @@ use std::task::Poll;
/// # Ok(())
/// # }
/// ```
#[derive(Debug)]
pub struct File {
std: std::fs::File,
std: Arc<sys::File>,
state: State,
/// Errors from writes/flushes are returned in write/flush calls. If a write
/// error is observed while performing a read, it is saved until the next
/// write / flush call.
last_write_err: Option<io::ErrorKind>,
}
#[derive(Debug)]
enum State {
Idle(Option<Buf>),
Busy(sys::Blocking<(Operation, Buf)>),
}
#[derive(Debug)]
enum Operation {
Read(io::Result<usize>),
Write(io::Result<()>),
Seek(io::Result<u64>),
}
impl File {
@@ -94,12 +118,12 @@ impl File {
/// ```
pub async fn open<P>(path: P) -> io::Result<File>
where
P: AsRef<Path> + Send + Unpin + 'static,
P: AsRef<Path>,
{
let mut open_options = OpenOptions::new();
open_options.read(true);
let path = path.as_ref().to_owned();
let std = asyncify(|| sys::File::open(path)).await?;
open_options.open(path).await
Ok(File::from_std(std))
}
/// Opens a file in write-only mode.
@@ -132,9 +156,10 @@ impl File {
/// ```
pub async fn create<P>(path: P) -> io::Result<File>
where
P: AsRef<Path> + Send + Unpin + 'static,
P: AsRef<Path>,
{
let std_file = asyncify(|| std::fs::File::create(&path)).await?;
let path = path.as_ref().to_owned();
let std_file = asyncify(move || sys::File::create(path)).await?;
Ok(File::from_std(std_file))
}
@@ -151,8 +176,12 @@ impl File {
/// let std_file = std::fs::File::open("foo.txt").unwrap();
/// let file = tokio::fs::File::from_std(std_file);
/// ```
pub fn from_std(std: std::fs::File) -> File {
File { std }
pub fn from_std(std: sys::File) -> File {
File {
std: Arc::new(std),
state: State::Idle(Some(Buf::with_capacity(0))),
last_write_err: None,
}
}
/// Seek to an offset, in bytes, in a stream.
@@ -174,8 +203,42 @@ impl File {
/// # Ok(())
/// # }
/// ```
pub async fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
asyncify(|| self.std.seek(pos)).await
pub async fn seek(&mut self, mut pos: SeekFrom) -> io::Result<u64> {
self.complete_inflight().await;
let mut buf = match self.state {
Idle(ref mut buf_cell) => buf_cell.take().unwrap(),
_ => unreachable!(),
};
// Factor in any unread data from the buf
if !buf.is_empty() {
let n = buf.discard_read();
if let SeekFrom::Current(ref mut offset) = pos {
*offset += n;
}
}
let std = self.std.clone();
// Start the operation
self.state = Busy(sys::run(move || {
let res = (&*std).seek(pos);
(Operation::Seek(res), buf)
}));
let (op, buf) = match self.state {
Idle(_) => unreachable!(),
Busy(ref mut rx) => rx.await,
};
self.state = Idle(Some(buf));
match op {
Operation::Seek(res) => res,
_ => unreachable!(),
}
}
/// Attempts to sync all OS-internal metadata to disk.
@@ -197,7 +260,10 @@ impl File {
/// # }
/// ```
pub async fn sync_all(&mut self) -> io::Result<()> {
asyncify(|| self.std.sync_all()).await
self.complete_inflight().await;
let std = self.std.clone();
asyncify(move || std.sync_all()).await
}
/// This function is similar to `poll_sync_all`, except that it may not
@@ -223,7 +289,10 @@ impl File {
/// # }
/// ```
pub async fn sync_data(&mut self) -> io::Result<()> {
asyncify(|| self.std.sync_data()).await
self.complete_inflight().await;
let std = self.std.clone();
asyncify(move || std.sync_data()).await
}
/// Truncates or extends the underlying file, updating the size of this file to become size.
@@ -252,7 +321,44 @@ impl File {
/// # }
/// ```
pub async fn set_len(&mut self, size: u64) -> io::Result<()> {
asyncify(|| self.std.set_len(size)).await
self.complete_inflight().await;
let mut buf = match self.state {
Idle(ref mut buf_cell) => buf_cell.take().unwrap(),
_ => unreachable!(),
};
let seek = if !buf.is_empty() {
Some(SeekFrom::Current(buf.discard_read()))
} else {
None
};
let std = self.std.clone();
self.state = Busy(sys::run(move || {
let res = if let Some(seek) = seek {
(&*std).seek(seek).and_then(|_| std.set_len(size))
} else {
std.set_len(size)
}
.map(|_| 0); // the value is discarded later
// Return the result as a seek
(Operation::Seek(res), buf)
}));
let (op, buf) = match self.state {
Idle(_) => unreachable!(),
Busy(ref mut rx) => rx.await,
};
self.state = Idle(Some(buf));
match op {
Operation::Seek(res) => res.map(|_| ()),
_ => unreachable!(),
}
}
/// Queries metadata about the underlying file.
@@ -271,7 +377,8 @@ impl File {
/// # }
/// ```
pub async fn metadata(&self) -> io::Result<Metadata> {
asyncify(|| self.std.metadata()).await
let std = self.std.clone();
asyncify(move || std.metadata()).await
}
/// Create a new `File` instance that shares the same underlying file handle
@@ -290,7 +397,8 @@ impl File {
/// # }
/// ```
pub async fn try_clone(&self) -> io::Result<File> {
let std_file = asyncify(|| self.std.try_clone()).await?;
let std = self.std.clone();
let std_file = asyncify(move || std.try_clone()).await?;
Ok(File::from_std(std_file))
}
@@ -324,54 +432,162 @@ impl File {
/// # }
/// ```
pub async fn set_permissions(&self, perm: Permissions) -> io::Result<()> {
asyncify(|| self.std.set_permissions(perm)).await
let std = self.std.clone();
asyncify(move || std.set_permissions(perm)).await
}
/// Destructures the `tokio_fs::File` into a [`std::fs::File`][std].
///
/// # Panics
///
/// This function will panic if `shutdown` has been called.
///
/// [std]: https://doc.rust-lang.org/std/fs/struct.File.html
///
/// # Examples
///
/// ```no_run
/// use tokio::fs::File;
///
/// # async fn dox() -> std::io::Result<()> {
/// let file = File::create("foo.txt").await?;
/// let std_file = file.into_std();
/// # Ok(())
/// # }
/// ```
pub fn into_std(self) -> std::fs::File {
self.std
async fn complete_inflight(&mut self) {
use futures_util::future::poll_fn;
if let Err(e) = poll_fn(|cx| Pin::new(&mut *self).poll_flush(cx)).await {
self.last_write_err = Some(e.kind());
}
}
}
impl AsyncRead for File {
fn poll_read(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut [u8],
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
dst: &mut [u8],
) -> Poll<io::Result<usize>> {
blocking_io(|| (&self.std).read(buf))
loop {
match self.state {
Idle(ref mut buf_cell) => {
let mut buf = buf_cell.take().unwrap();
if !buf.is_empty() {
let n = buf.copy_to(dst);
*buf_cell = Some(buf);
return Ready(Ok(n));
}
buf.ensure_capacity_for(dst);
let std = self.std.clone();
self.state = Busy(sys::run(move || {
let res = buf.read_from(&mut &*std);
(Operation::Read(res), buf)
}));
}
Busy(ref mut rx) => {
let (op, mut buf) = ready!(Pin::new(rx).poll(cx));
match op {
Operation::Read(Ok(_)) => {
let n = buf.copy_to(dst);
self.state = Idle(Some(buf));
return Ready(Ok(n));
}
Operation::Read(Err(e)) => {
assert!(buf.is_empty());
self.state = Idle(Some(buf));
return Ready(Err(e));
}
Operation::Write(Ok(_)) => {
assert!(buf.is_empty());
self.state = Idle(Some(buf));
continue;
}
Operation::Write(Err(e)) => {
assert!(self.last_write_err.is_none());
self.last_write_err = Some(e.kind());
self.state = Idle(Some(buf));
}
Operation::Seek(_) => {
assert!(buf.is_empty());
self.state = Idle(Some(buf));
continue;
}
}
}
}
}
}
}
impl AsyncWrite for File {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &[u8],
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
src: &[u8],
) -> Poll<io::Result<usize>> {
blocking_io(|| (&self.std).write(buf))
if let Some(e) = self.last_write_err.take() {
return Ready(Err(e.into()));
}
loop {
match self.state {
Idle(ref mut buf_cell) => {
let mut buf = buf_cell.take().unwrap();
let seek = if !buf.is_empty() {
Some(SeekFrom::Current(buf.discard_read()))
} else {
None
};
let n = buf.copy_from(src);
let std = self.std.clone();
self.state = Busy(sys::run(move || {
let res = if let Some(seek) = seek {
(&*std).seek(seek).and_then(|_| buf.write_to(&mut &*std))
} else {
buf.write_to(&mut &*std)
};
(Operation::Write(res), buf)
}));
return Ready(Ok(n));
}
Busy(ref mut rx) => {
let (op, buf) = ready!(Pin::new(rx).poll(cx));
self.state = Idle(Some(buf));
match op {
Operation::Read(_) => {
// We don't care about the result here. The fact
// that the cursor has advanced will be reflected in
// the next iteration of the loop
continue;
}
Operation::Write(res) => {
// If the previous write was successful, continue.
// Otherwise, error.
res?;
continue;
}
Operation::Seek(_) => {
// Ignore the seek
continue;
}
}
}
}
}
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
blocking_io(|| (&self.std).flush())
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
if let Some(e) = self.last_write_err.take() {
return Ready(Err(e.into()));
}
let (op, buf) = match self.state {
Idle(_) => return Ready(Ok(())),
Busy(ref mut rx) => ready!(Pin::new(rx).poll(cx)),
};
// The buffer is not used here
self.state = Idle(Some(buf));
match op {
Operation::Read(_) => Ready(Ok(())),
Operation::Write(res) => Ready(res),
Operation::Seek(_) => Ready(Ok(())),
}
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
@@ -379,16 +595,16 @@ impl AsyncWrite for File {
}
}
impl From<std::fs::File> for File {
fn from(std: std::fs::File) -> Self {
impl From<sys::File> for File {
fn from(std: sys::File) -> Self {
Self::from_std(std)
}
}
impl TryFrom<File> for std::fs::File {
type Error = io::Error;
fn try_from(file: File) -> Result<Self, Self::Error> {
Ok(file.std)
impl fmt::Debug for File {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.debug_struct("tokio::fs::File")
.field("std", &self.std)
.finish()
}
}
tokio-fs/src/hard_link.rs
+4 -1
View File
@@ -12,5 +12,8 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.hard_link.html
pub async fn hard_link<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) -> io::Result<()> {
asyncify(|| std::fs::hard_link(&src, &dst)).await
let src = src.as_ref().to_owned();
let dst = dst.as_ref().to_owned();
asyncify(move || std::fs::hard_link(src, dst)).await
}
tokio-fs/src/lib.rs
+9 -27
View File
@@ -34,6 +34,7 @@
//! [`AsyncRead`]: https://docs.rs/tokio-io/0.1/tokio_io/trait.AsyncRead.html
//! [tokio-executor]: https://docs.rs/tokio-executor/0.2.0-alpha.2/tokio_executor/threadpool/index.html
mod blocking;
mod create_dir;
mod create_dir_all;
mod file;
@@ -76,37 +77,18 @@ pub use crate::symlink_metadata::symlink_metadata;
pub use crate::write::write;
use std::io;
use std::io::ErrorKind::Other;
use std::task::Poll;
use std::task::Poll::*;
fn blocking_io<F, T>(f: F) -> Poll<io::Result<T>>
where
F: FnOnce() -> io::Result<T>,
{
use tokio_executor::threadpool::blocking;
match blocking(f) {
Ready(Ok(v)) => Ready(v),
Ready(Err(_)) => Ready(Err(blocking_err())),
Pending => Pending,
}
}
async fn asyncify<F, T>(f: F) -> io::Result<T>
where
F: FnOnce() -> io::Result<T>,
F: FnOnce() -> io::Result<T> + Send + 'static,
T: Send + 'static,
{
use futures_util::future::poll_fn;
let mut f = Some(f);
poll_fn(move |_| blocking_io(|| f.take().unwrap()())).await
sys::run(f).await
}
fn blocking_err() -> io::Error {
io::Error::new(
Other,
"`blocking` annotated I/O must be called \
from the context of the Tokio runtime.",
)
/// Types in this module can be mocked out in tests.
mod sys {
pub(crate) use std::fs::File;
pub(crate) use tokio_executor::blocking::{run, Blocking};
}
tokio-fs/src/metadata.rs
+3 -2
View File
@@ -7,7 +7,8 @@ use std::path::Path;
/// Queries the file system metadata for a path.
pub async fn metadata<P>(path: P) -> io::Result<Metadata>
where
P: AsRef<Path> + Send + 'static,
P: AsRef<Path>,
{
asyncify(|| std::fs::metadata(&path)).await
let path = path.as_ref().to_owned();
asyncify(|| std::fs::metadata(path)).await
}
tokio-fs/src/open_options.rs
+7 -5
View File
@@ -1,6 +1,5 @@
use super::File;
use crate::{asyncify, File};
use futures_util::future::poll_fn;
use std::io;
use std::path::Path;
@@ -91,10 +90,13 @@ impl OpenOptions {
/// [`open`]: https://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.open
pub async fn open<P>(&self, path: P) -> io::Result<File>
where
P: AsRef<Path> + Send + Unpin + 'static,
P: AsRef<Path>,
{
let std_file = poll_fn(|_| crate::blocking_io(|| self.0.open(&path))).await?;
Ok(File::from_std(std_file))
let path = path.as_ref().to_owned();
let opts = self.0.clone();
let std = asyncify(move || opts.open(path)).await?;
Ok(File::from_std(std))
}
}
tokio-fs/src/os/unix/symlink.rs
+4 -1
View File
@@ -11,5 +11,8 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/os/unix/fs/fn.symlink.html
pub async fn symlink<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) -> io::Result<()> {
asyncify(|| std::os::unix::fs::symlink(&src, &dst)).await
let src = src.as_ref().to_owned();
let dst = dst.as_ref().to_owned();
asyncify(move || std::os::unix::fs::symlink(src, dst)).await
}
tokio-fs/src/os/windows/symlink_dir.rs
+4 -1
View File
@@ -12,5 +12,8 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/os/windows/fs/fn.symlink_dir.html
pub async fn symlink_dir<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) -> io::Result<()> {
asyncify(|| std::os::windows::fs::symlink_dir(&src, &dst)).await
let src = src.as_ref().to_owned();
let dst = dst.as_ref().to_owned();
asyncify(move || std::os::windows::fs::symlink_dir(src, dst)).await
}
tokio-fs/src/os/windows/symlink_file.rs
+4 -1
View File
@@ -12,5 +12,8 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/os/windows/fs/fn.symlink_file.html
pub async fn symlink_file<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) -> io::Result<()> {
asyncify(|| std::os::windows::fs::symlink_file(&src, &dst)).await
let src = src.as_ref().to_owned();
let dst = dst.as_ref().to_owned();
asyncify(move || std::os::windows::fs::symlink_file(src, dst)).await
}
tokio-fs/src/read.rs
+4 -10
View File
@@ -1,6 +1,4 @@
use crate::File;
use tokio_io::AsyncReadExt;
use crate::asyncify;
use std::{io, path::Path};
@@ -22,12 +20,8 @@ use std::{io, path::Path};
/// ```
pub async fn read<P>(path: P) -> io::Result<Vec<u8>>
where
P: AsRef<Path> + Send + Unpin + 'static,
P: AsRef<Path>,
{
let mut file = File::open(path).await?;
let metadata = file.metadata().await?;
let mut contents = Vec::with_capacity(metadata.len() as usize + 1);
file.read_to_end(&mut contents).await?;
Ok(contents)
let path = path.as_ref().to_owned();
asyncify(move || std::fs::read(path)).await
}
tokio-fs/src/read_dir.rs
+40 -25
View File
@@ -1,13 +1,16 @@
use crate::{asyncify, blocking_io};
use crate::{asyncify, sys};
use futures_core::ready;
use futures_core::stream::Stream;
use std::ffi::OsString;
use std::fs::{DirEntry as StdDirEntry, FileType, Metadata};
use std::fs::{FileType, Metadata};
use std::future::Future;
use std::io;
#[cfg(unix)]
use std::os::unix::fs::DirEntryExt;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
@@ -20,8 +23,10 @@ pub async fn read_dir<P>(path: P) -> io::Result<ReadDir>
where
P: AsRef<Path> + Send + 'static,
{
let std = asyncify(|| std::fs::read_dir(&path)).await?;
Ok(ReadDir(std))
let path = path.as_ref().to_owned();
let std = asyncify(|| std::fs::read_dir(path)).await?;
Ok(ReadDir(State::Idle(Some(std))))
}
/// Stream of the entries in a directory.
@@ -42,22 +47,37 @@ where
/// [`Err`]: https://doc.rust-lang.org/std/result/enum.Result.html#variant.Err
#[derive(Debug)]
#[must_use = "streams do nothing unless polled"]
pub struct ReadDir(std::fs::ReadDir);
pub struct ReadDir(State);
#[derive(Debug)]
enum State {
Idle(Option<std::fs::ReadDir>),
Pending(sys::Blocking<(Option<io::Result<std::fs::DirEntry>>, std::fs::ReadDir)>),
}
impl Stream for ReadDir {
type Item = io::Result<DirEntry>;
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let res = blocking_io(|| match self.0.next() {
Some(Err(err)) => Err(err),
Some(Ok(item)) => Ok(Some(Ok(DirEntry(item)))),
None => Ok(None),
});
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
loop {
match self.0 {
State::Idle(ref mut std) => {
let mut std = std.take().unwrap();
match res {
Poll::Ready(Err(err)) => Poll::Ready(Some(Err(err))),
Poll::Ready(Ok(v)) => Poll::Ready(v),
Poll::Pending => Poll::Pending,
self.0 = State::Pending(sys::run(move || {
let ret = std.next();
(ret, std)
}));
}
State::Pending(ref mut rx) => {
let (ret, std) = ready!(Pin::new(rx).poll(cx));
self.0 = State::Idle(Some(std));
let ret = ret.map(|res| res.map(|std| DirEntry(Arc::new(std))));
return Poll::Ready(ret);
}
}
}
}
}
@@ -75,16 +95,9 @@ impl Stream for ReadDir {
///
/// [std]: https://doc.rust-lang.org/std/fs/struct.DirEntry.html
#[derive(Debug)]
pub struct DirEntry(StdDirEntry);
pub struct DirEntry(Arc<std::fs::DirEntry>);
impl DirEntry {
/// Destructures the `tokio_fs::DirEntry` into a [`std::fs::DirEntry`][std].
///
/// [std]: https://doc.rust-lang.org/std/fs/struct.DirEntry.html
pub fn into_std(self) -> StdDirEntry {
self.0
}
/// Returns the full path to the file that this entry represents.
///
/// The full path is created by joining the original path to `read_dir`
@@ -177,7 +190,8 @@ impl DirEntry {
/// # }
/// ```
pub async fn metadata(&self) -> io::Result<Metadata> {
asyncify(|| self.0.metadata()).await
let std = self.0.clone();
asyncify(move || std.metadata()).await
}
/// Return the file type for the file that this entry points at.
@@ -214,7 +228,8 @@ impl DirEntry {
/// # }
/// ```
pub async fn file_type(&self) -> io::Result<FileType> {
asyncify(|| self.0.file_type()).await
let std = self.0.clone();
asyncify(move || std.file_type()).await
}
}
tokio-fs/src/read_link.rs
+2 -1
View File
@@ -9,5 +9,6 @@ use std::path::{Path, PathBuf};
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.read_link.html
pub async fn read_link<P: AsRef<Path>>(path: P) -> io::Result<PathBuf> {
asyncify(|| std::fs::read_link(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::read_link(path)).await
}
tokio-fs/src/remove_dir.rs
+2 -1
View File
@@ -9,5 +9,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.remove_dir.html
pub async fn remove_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
asyncify(|| std::fs::remove_dir(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::remove_dir(path)).await
}
tokio-fs/src/remove_dir_all.rs
+2 -1
View File
@@ -9,5 +9,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.remove_dir_all.html
pub async fn remove_dir_all<P: AsRef<Path>>(path: P) -> io::Result<()> {
asyncify(|| std::fs::remove_dir_all(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::remove_dir_all(path)).await
}
tokio-fs/src/remove_file.rs
+2 -1
View File
@@ -13,5 +13,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.remove_file.html
pub async fn remove_file<P: AsRef<Path>>(path: P) -> io::Result<()> {
asyncify(|| std::fs::remove_file(&path)).await
let path = path.as_ref().to_owned();
asyncify(move || std::fs::remove_file(path)).await
}
tokio-fs/src/rename.rs
+4 -1
View File
@@ -12,5 +12,8 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.rename.html
pub async fn rename<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> io::Result<()> {
asyncify(|| std::fs::rename(&from, &to)).await
let from = from.as_ref().to_owned();
let to = to.as_ref().to_owned();
asyncify(move || std::fs::rename(from, to)).await
}
tokio-fs/src/set_permissions.rs
+2 -1
View File
@@ -10,5 +10,6 @@ use std::path::Path;
///
/// [std]: https://doc.rust-lang.org/std/fs/fn.set_permissions.html
pub async fn set_permissions<P: AsRef<Path>>(path: P, perm: Permissions) -> io::Result<()> {
asyncify(|| std::fs::set_permissions(&path, perm)).await
let path = path.as_ref().to_owned();
asyncify(|| std::fs::set_permissions(path, perm)).await
}
tokio-fs/src/stderr.rs
+15 -10
View File
@@ -1,8 +1,8 @@
use crate::blocking_io;
use crate::blocking::Blocking;
use tokio_io::AsyncWrite;
use std::io::{self, Write};
use std::io;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -18,7 +18,7 @@ use std::task::Poll;
/// [`AsyncWrite`]: trait.AsyncWrite.html
#[derive(Debug)]
pub struct Stderr {
std: std::io::Stderr,
std: Blocking<std::io::Stderr>,
}
/// Constructs a new handle to the standard error of the current process.
@@ -27,23 +27,28 @@ pub struct Stderr {
/// Tokio runtime.
pub fn stderr() -> Stderr {
let std = io::stderr();
Stderr { std }
Stderr {
std: Blocking::new(std),
}
}
impl AsyncWrite for Stderr {
fn poll_write(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
blocking_io(|| (&mut self.std).write(buf))
Pin::new(&mut self.std).poll_write(cx, buf)
}
fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
blocking_io(|| (&mut self.std).flush())
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.std).poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
fn poll_shutdown(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.std).poll_shutdown(cx)
}
}
tokio-fs/src/stdin.rs
+8 -6
View File
@@ -1,8 +1,8 @@
use crate::blocking_io;
use crate::blocking::Blocking;
use tokio_io::AsyncRead;
use std::io::{self, Read};
use std::io;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -24,7 +24,7 @@ use std::task::Poll;
/// [`AsyncRead`]: trait.AsyncRead.html
#[derive(Debug)]
pub struct Stdin {
std: std::io::Stdin,
std: Blocking<std::io::Stdin>,
}
/// Constructs a new handle to the standard input of the current process.
@@ -33,15 +33,17 @@ pub struct Stdin {
/// Tokio runtime.
pub fn stdin() -> Stdin {
let std = io::stdin();
Stdin { std }
Stdin {
std: Blocking::new(std),
}
}
impl AsyncRead for Stdin {
fn poll_read(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
cx: &mut Context<'_>,
buf: &mut [u8],
) -> Poll<io::Result<usize>> {
blocking_io(|| (&mut self.std).read(buf))
Pin::new(&mut self.std).poll_read(cx, buf)
}
}
tokio-fs/src/stdout.rs
+15 -10
View File
@@ -1,8 +1,8 @@
use crate::blocking_io;
use crate::blocking::Blocking;
use tokio_io::AsyncWrite;
use std::io::{self, Write};
use std::io;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -18,7 +18,7 @@ use std::task::Poll;
/// [`AsyncWrite`]: trait.AsyncWrite.html
#[derive(Debug)]
pub struct Stdout {
std: std::io::Stdout,
std: Blocking<std::io::Stdout>,
}
/// Constructs a new handle to the standard output of the current process.
@@ -27,23 +27,28 @@ pub struct Stdout {
/// runtime.
pub fn stdout() -> Stdout {
let std = io::stdout();
Stdout { std }
Stdout {
std: Blocking::new(std),
}
}
impl AsyncWrite for Stdout {
fn poll_write(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
blocking_io(|| (&mut self.std).write(buf))
Pin::new(&mut self.std).poll_write(cx, buf)
}
fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
blocking_io(|| (&mut self.std).flush())
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.std).poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Poll::Ready(Ok(()))
fn poll_shutdown(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.std).poll_shutdown(cx)
}
}
tokio-fs/src/symlink_metadata.rs
+2 -1
View File
@@ -13,5 +13,6 @@ pub async fn symlink_metadata<P>(path: P) -> io::Result<Metadata>
where
P: AsRef<Path> + Send + 'static,
{
asyncify(|| std::fs::symlink_metadata(&path)).await
let path = path.as_ref().to_owned();
asyncify(|| std::fs::symlink_metadata(path)).await
}
tokio-fs/src/write.rs
+4 -6
View File
@@ -1,6 +1,4 @@
use crate::File;
use tokio_io::AsyncWriteExt;
use crate::asyncify;
use std::{io, path::Path};
@@ -23,8 +21,8 @@ pub async fn write<P, C: AsRef<[u8]> + Unpin>(path: P, contents: C) -> io::Resul
where
P: AsRef<Path> + Send + Unpin + 'static,
{
let mut file = File::create(path).await?;
file.write_all(contents.as_ref()).await?;
let path = path.as_ref().to_owned();
let contents = contents.as_ref().to_owned();
Ok(())
asyncify(move || std::fs::write(path, contents)).await
}