add experimental-async feature (#3931)

* add `experimental-async` feature

* gate async doctests on feature

Cargo.toml

@@ -65,6 +65,9 @@ pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", featu
 [features]
 default = ["macros"]
 
+# Enables support for `async fn` for `#[pyfunction]` and `#[pymethods]`.
+experimental-async = ["macros", "pyo3-macros/experimental-async"]
+
 # Enables pyo3::inspect module and additional type information on FromPyObject
 # and IntoPy traits
 experimental-inspect = []
@@ -115,8 +118,9 @@ full = [
     "chrono",
     "chrono-tz",
     "either",
-    "experimental-inspect",
+    "experimental-async",
     "experimental-declarative-modules",
+    "experimental-inspect",
     "eyre",
     "hashbrown",
     "indexmap",

guide/src/async-await.md

@@ -6,6 +6,7 @@
 
 ```rust
 # #![allow(dead_code)]
+# #[cfg(feature = "experimental-async")] {
 use std::{thread, time::Duration};
 use futures::channel::oneshot;
 use pyo3::prelude::*;
@@ -20,6 +21,7 @@ async fn sleep(seconds: f64, result: Option<PyObject>) -> Option<PyObject> {
     rx.await.unwrap();
     result
 }
+# }
 ```
 
 *Python awaitables instantiated with this method can only be awaited in *asyncio* context. Other Python async runtime may be supported in the future.*
@@ -72,6 +74,7 @@ Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOC
 
 ```rust
 # #![allow(dead_code)]
+# #[cfg(feature = "experimental-async")] {
 use futures::FutureExt;
 use pyo3::prelude::*;
 use pyo3::coroutine::CancelHandle;
@@ -83,11 +86,12 @@ async fn cancellable(#[pyo3(cancel_handle)] mut cancel: CancelHandle) {
         _ = cancel.cancelled().fuse() => println!("cancelled"),
     }
 }
+# }
 ```
 
 ## The `Coroutine` type
 
-To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). 
+To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine).
 
 Each `coroutine.send` call is translated to a `Future::poll` call. If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised;
 

guide/src/features.md

@@ -51,6 +51,12 @@ If you do not enable this feature, you should call `pyo3::prepare_freethreaded_p
 
 ## Advanced Features
 
+### `experimental-async`
+
+This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`.
+
+The feature has some unfinished refinements and performance improvements. To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs.
+
 ### `experimental-inspect`
 
 This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.

newsfragments/3931.added.md

@@ -0,0 +1 @@
+Add `experimental-async` feature.

pyo3-macros-backend/Cargo.toml

@@ -26,3 +26,6 @@ features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-trait
 
 [lints]
 workspace = true
+
+[features]
+experimental-async = []

pyo3-macros-backend/src/method.rs

@@ -512,6 +512,13 @@ impl<'a> FnSpec<'a> {
             }
         }
 
+        if self.asyncness.is_some() {
+            ensure_spanned!(
+                cfg!(feature = "experimental-async"),
+                self.asyncness.span() => "async functions are only supported with the `experimental-async` feature"
+            );
+        }
+
         let rust_call = |args: Vec<TokenStream>, holders: &mut Vec<TokenStream>| {
             let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx);
 

pyo3-macros/Cargo.toml

@@ -15,6 +15,7 @@ proc-macro = true
 
 [features]
 multiple-pymethods = []
+experimental-async = ["pyo3-macros-backend/experimental-async"]
 experimental-declarative-modules = []
 
 [dependencies]

src/impl_.rs

@@ -6,7 +6,7 @@
 //! APIs may may change at any time without documentation in the CHANGELOG and without
 //! breaking semver guarantees.
 
-#[cfg(feature = "macros")]
+#[cfg(feature = "experimental-async")]
 pub mod coroutine;
 pub mod deprecations;
 pub mod extract_argument;

src/lib.rs

@@ -418,7 +418,7 @@ pub mod buffer;
 pub mod callback;
 pub mod conversion;
 mod conversions;
-#[cfg(feature = "macros")]
+#[cfg(feature = "experimental-async")]
 pub mod coroutine;
 #[macro_use]
 #[doc(hidden)]

tests/test_compile_error.rs

@@ -27,7 +27,8 @@ fn test_compile_errors() {
     t.compile_fail("tests/ui/wrong_aspyref_lifetimes.rs");
     t.compile_fail("tests/ui/invalid_pyfunctions.rs");
     t.compile_fail("tests/ui/invalid_pymethods.rs");
-    #[cfg(Py_LIMITED_API)]
+    // output changes with async feature
+    #[cfg(all(Py_LIMITED_API, feature = "experimental-async"))]
     t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs");
     t.compile_fail("tests/ui/invalid_intern_arg.rs");
     t.compile_fail("tests/ui/invalid_frozen_pyclass_borrow.rs");
@@ -48,4 +49,6 @@ fn test_compile_errors() {
     t.compile_fail("tests/ui/invalid_pymodule_trait.rs");
     #[cfg(feature = "experimental-declarative-modules")]
     t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs");
+    #[cfg(feature = "experimental-async")]
+    t.compile_fail("tests/ui/invalid_cancel_handle.rs");
 }

tests/test_coroutine.rs

@@ -1,4 +1,4 @@
-#![cfg(feature = "macros")]
+#![cfg(feature = "experimental-async")]
 #![cfg(not(target_arch = "wasm32"))]
 use std::{task::Poll, thread, time::Duration};
 

tests/ui/invalid_argument_attributes.rs

@@ -15,29 +15,4 @@ fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] param: String) {
 #[pyfunction]
 fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {}
 
-#[pyfunction]
-async fn from_py_with_value_and_cancel_handle(
-    #[pyo3(from_py_with = "func", cancel_handle)] _param: String,
-) {
-}
-
-#[pyfunction]
-async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {}
-
-#[pyfunction]
-async fn cancel_handle_repeated2(
-    #[pyo3(cancel_handle)] _param: String,
-    #[pyo3(cancel_handle)] _param2: String,
-) {
-}
-
-#[pyfunction]
-fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {}
-
-#[pyfunction]
-async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {}
-
-#[pyfunction]
-async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
-
 fn main() {}

tests/ui/invalid_argument_attributes.stderr

@@ -27,82 +27,3 @@ error: `from_py_with` may only be specified once per argument
    |
 16 | fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {}
    |                                                        ^^^^^^^^^^^^
-
-error: `from_py_with` and `cancel_handle` cannot be specified together
-  --> tests/ui/invalid_argument_attributes.rs:20:35
-   |
-20 |     #[pyo3(from_py_with = "func", cancel_handle)] _param: String,
-   |                                   ^^^^^^^^^^^^^
-
-error: `cancel_handle` may only be specified once per argument
-  --> tests/ui/invalid_argument_attributes.rs:25:55
-   |
-25 | async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {}
-   |                                                       ^^^^^^^^^^^^^
-
-error: `cancel_handle` may only be specified once
-  --> tests/ui/invalid_argument_attributes.rs:30:28
-   |
-30 |     #[pyo3(cancel_handle)] _param2: String,
-   |                            ^^^^^^^
-
-error: `cancel_handle` attribute can only be used with `async fn`
-  --> tests/ui/invalid_argument_attributes.rs:35:53
-   |
-35 | fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {}
-   |                                                     ^^^^^^
-
-error[E0308]: mismatched types
-  --> tests/ui/invalid_argument_attributes.rs:37:1
-   |
-37 | #[pyfunction]
-   | ^^^^^^^^^^^^^
-   | |
-   | expected `String`, found `CancelHandle`
-   | arguments to this function are incorrect
-   |
-note: function defined here
-  --> tests/ui/invalid_argument_attributes.rs:38:10
-   |
-38 | async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {}
-   |          ^^^^^^^^^^^^^^^^^^^^^^^^                        --------------
-   = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info)
-
-error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied
-  --> tests/ui/invalid_argument_attributes.rs:41:50
-   |
-41 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
-   |                                                  ^^^^ the trait `PyClass` is not implemented for `CancelHandle`
-   |
-   = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine`
-   = note: required for `CancelHandle` to implement `FromPyObject<'_>`
-   = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>`
-note: required by a bound in `extract_argument`
-  --> src/impl_/extract_argument.rs
-   |
-   | pub fn extract_argument<'a, 'py, T>(
-   |        ---------------- required by a bound in this function
-...
-   |     T: PyFunctionArgument<'a, 'py>,
-   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument`
-
-error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied
-  --> tests/ui/invalid_argument_attributes.rs:41:50
-   |
-41 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
-   |                                                  ^^^^ the trait `Clone` is not implemented for `CancelHandle`
-   |
-   = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`:
-             &'a pyo3::Bound<'py, T>
-             &'a pyo3::coroutine::Coroutine
-             &'a mut pyo3::coroutine::Coroutine
-   = note: required for `CancelHandle` to implement `FromPyObject<'_>`
-   = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>`
-note: required by a bound in `extract_argument`
-  --> src/impl_/extract_argument.rs
-   |
-   | pub fn extract_argument<'a, 'py, T>(
-   |        ---------------- required by a bound in this function
-...
-   |     T: PyFunctionArgument<'a, 'py>,
-   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument`

tests/ui/invalid_cancel_handle.rs

@@ -0,0 +1,22 @@
+use pyo3::prelude::*;
+
+#[pyfunction]
+async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {}
+
+#[pyfunction]
+async fn cancel_handle_repeated2(
+    #[pyo3(cancel_handle)] _param: String,
+    #[pyo3(cancel_handle)] _param2: String,
+) {
+}
+
+#[pyfunction]
+fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {}
+
+#[pyfunction]
+async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {}
+
+#[pyfunction]
+async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
+
+fn main() {}

tests/ui/invalid_cancel_handle.stderr

@@ -0,0 +1,72 @@
+error: `cancel_handle` may only be specified once per argument
+ --> tests/ui/invalid_cancel_handle.rs:4:55
+  |
+4 | async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {}
+  |                                                       ^^^^^^^^^^^^^
+
+error: `cancel_handle` may only be specified once
+ --> tests/ui/invalid_cancel_handle.rs:9:28
+  |
+9 |     #[pyo3(cancel_handle)] _param2: String,
+  |                            ^^^^^^^
+
+error: `cancel_handle` attribute can only be used with `async fn`
+  --> tests/ui/invalid_cancel_handle.rs:14:53
+   |
+14 | fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {}
+   |                                                     ^^^^^^
+
+error[E0308]: mismatched types
+  --> tests/ui/invalid_cancel_handle.rs:16:1
+   |
+16 | #[pyfunction]
+   | ^^^^^^^^^^^^^
+   | |
+   | expected `String`, found `CancelHandle`
+   | arguments to this function are incorrect
+   |
+note: function defined here
+  --> tests/ui/invalid_cancel_handle.rs:17:10
+   |
+17 | async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {}
+   |          ^^^^^^^^^^^^^^^^^^^^^^^^                        --------------
+   = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied
+  --> tests/ui/invalid_cancel_handle.rs:20:50
+   |
+20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
+   |                                                  ^^^^ the trait `PyClass` is not implemented for `CancelHandle`
+   |
+   = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine`
+   = note: required for `CancelHandle` to implement `FromPyObject<'_>`
+   = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>`
+note: required by a bound in `extract_argument`
+  --> src/impl_/extract_argument.rs
+   |
+   | pub fn extract_argument<'a, 'py, T>(
+   |        ---------------- required by a bound in this function
+...
+   |     T: PyFunctionArgument<'a, 'py>,
+   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument`
+
+error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied
+  --> tests/ui/invalid_cancel_handle.rs:20:50
+   |
+20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {}
+   |                                                  ^^^^ the trait `Clone` is not implemented for `CancelHandle`
+   |
+   = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`:
+             &'a pyo3::Bound<'py, T>
+             &'a pyo3::coroutine::Coroutine
+             &'a mut pyo3::coroutine::Coroutine
+   = note: required for `CancelHandle` to implement `FromPyObject<'_>`
+   = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>`
+note: required by a bound in `extract_argument`
+  --> src/impl_/extract_argument.rs
+   |
+   | pub fn extract_argument<'a, 'py, T>(
+   |        ---------------- required by a bound in this function
+...
+   |     T: PyFunctionArgument<'a, 'py>,
+   |        ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument`