Fix a deadlock that can happen when shutting down (#409)

There is a deadlock that can occur when the concurrent runtime shuts
down. This patch adds a test and fix.

Fixes #401.

tokio-reactor/src/atomic_task.rs

@@ -237,10 +237,9 @@ impl AtomicTask {
         }
     }
 
-    /// Notifies the task that last called `register`.
-    ///
-    /// If `register` has not been called yet, then this does nothing.
-    pub fn notify(&self) {
+    /// Attempts to take the `Task` value out of the `AtomicTask` with the
+    /// intention that the caller will notify the task.
+    pub fn take_to_notify(&self) -> Option<Task> {
         // AcqRel ordering is used in order to acquire the value of the `task`
         // cell as well as to establish a `release` ordering with whatever
         // memory the `AtomicTask` is associated with.
@@ -252,9 +251,7 @@ impl AtomicTask {
                 // Release the lock
                 self.state.fetch_and(!NOTIFYING, Release);
 
-                if let Some(task) = task {
-                    task.notify();
-                }
+                task
             }
             state => {
                 // There is a concurrent thread currently updating the
@@ -268,9 +265,20 @@ impl AtomicTask {
                     state == REGISTERING ||
                     state == REGISTERING | NOTIFYING ||
                     state == NOTIFYING);
+
+                None
             }
         }
     }
+
+    /// Notifies the task that last called `register`.
+    ///
+    /// If `register` has not been called yet, then this does nothing.
+    pub fn notify(&self) {
+        if let Some(task) = self.take_to_notify() {
+            task.notify();
+        }
+    }
 }
 
 impl Default for AtomicTask {

tokio-reactor/src/lib.rs

@@ -392,9 +392,19 @@ impl Reactor {
         let aba_guard = token.0 & !MAX_SOURCES;
         let token = token.0 & MAX_SOURCES;
 
-        let io_dispatch = self.inner.io_dispatch.read().unwrap();
+        let mut rd = None;
+        let mut wr = None;
+
+        // Create a scope to ensure that notifying the tasks stays out of the
+        // lock's critical section.
+        {
+            let io_dispatch = self.inner.io_dispatch.read().unwrap();
+
+            let io = match io_dispatch.get(token) {
+                Some(io) => io,
+                None => return,
+            };
 
-        if let Some(io) = io_dispatch.get(token) {
             if aba_guard != io.aba_guard {
                 return;
             }
@@ -402,13 +412,21 @@ impl Reactor {
             io.readiness.fetch_or(ready.as_usize(), Relaxed);
 
             if ready.is_writable() || platform::is_hup(&ready) {
-                io.writer.notify();
+                wr = io.writer.take_to_notify();
             }
 
             if !(ready & (!mio::Ready::writable())).is_empty() {
-                io.reader.notify();
+                rd = io.reader.take_to_notify();
             }
         }
+
+        if let Some(task) = rd {
+            task.notify();
+        }
+
+        if let Some(task) = wr {
+            task.notify();
+        }
     }
 }