Be more principled about when blocking is ok (#2410)

This enables `block_in_place` to be used in more contexts. Specifically,
it allows you to block whenever you are off the tokio runtime (like if
you are not using tokio, are in a `spawn_blocking` closure, etc.), and
in the threaded scheduler's `block_on`. Blocking in `LocalSet` and the
basic scheduler's` block_on` is still disallowed.

Fixes #2327.
Fixes #2393.

tokio/tests/task_blocking.rs

@@ -1,7 +1,7 @@
 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 
-use tokio::task;
+use tokio::{runtime, task};
 use tokio_test::assert_ok;
 
 use std::thread;
@@ -28,6 +28,29 @@ async fn basic_blocking() {
     }
 }
 
+#[tokio::test(threaded_scheduler)]
+async fn block_in_blocking() {
+    // Run a few times
+    for _ in 0..100 {
+        let out = assert_ok!(
+            tokio::spawn(async {
+                assert_ok!(
+                    task::spawn_blocking(|| {
+                        task::block_in_place(|| {
+                            thread::sleep(Duration::from_millis(5));
+                        });
+                        "hello"
+                    })
+                    .await
+                )
+            })
+            .await
+        );
+
+        assert_eq!(out, "hello");
+    }
+}
+
 #[tokio::test(threaded_scheduler)]
 async fn block_in_block() {
     // Run a few times
@@ -47,3 +70,29 @@ async fn block_in_block() {
         assert_eq!(out, "hello");
     }
 }
+
+#[tokio::test(basic_scheduler)]
+#[should_panic]
+async fn no_block_in_basic_scheduler() {
+    task::block_in_place(|| {});
+}
+
+#[test]
+fn yes_block_in_threaded_block_on() {
+    let mut rt = runtime::Builder::new()
+        .threaded_scheduler()
+        .build()
+        .unwrap();
+    rt.block_on(async {
+        task::block_in_place(|| {});
+    });
+}
+
+#[test]
+#[should_panic]
+fn no_block_in_basic_block_on() {
+    let mut rt = runtime::Builder::new().basic_scheduler().build().unwrap();
+    rt.block_on(async {
+        task::block_in_place(|| {});
+    });
+}