WIP [internal] Allow going from Rust-CPython pointers to PyO3 pyclass structs

src/rust/engine/Cargo.toml

@@ -25,6 +25,7 @@ members = [
   "client",
   "concrete_time",
   "engine_pyo3",
+  "engine_pyo3_extension",
   "fs",
   "fs/brfs",
   "fs/fs_util",
@@ -65,6 +66,7 @@ default-members = [
   "async_value",
   "concrete_time",
   "engine_pyo3",
+  "engine_pyo3_extension",
   "fs",
   "fs/fs_util",
   "fs/store",
@@ -126,6 +128,7 @@ num_enum = "0.4"
 parking_lot = "0.11"
 petgraph = "0.5"
 process_execution = { path = "process_execution" }
+pyo3 = "0.14.1"
 rand = "0.8"
 regex = "1"
 reqwest = { version = "0.11", default_features = false, features = ["stream", "rustls-tls"] }

src/rust/engine/engine_pyo3/Cargo.toml

@@ -5,18 +5,6 @@ name = "engine_pyo3"
 authors = [ "Pants Build <[email protected]>" ]
 publish = false
 
-[lib]
-crate-type = ["cdylib"]
-
-[features]
-# NB: To actually load this crate from python, the `extension-module` feature must be enabled. But
-# unfortunately, enabling `extension-module` causes tests linked against `pyo3` to fail. We
-# define a feature to enable that, but we do not enable it by default: someone building this module
-# in order to extract `libengine_pyo3.so` should pass `cargo build .. --features=engine_pyo3/extension-module`.
-#  see https://github.com/PyO3/pyo3/issues/340
-extension-module = ["pyo3/extension-module"]
-default = []
-
 [dependencies]
 fs = { path = "../fs" }
 hashing = { path = "../hashing" }

src/rust/engine/engine_pyo3/src/externs/interface/fs.rs → src/rust/engine/engine_pyo3/src/fs.rs

@@ -14,7 +14,7 @@ use fs::{GlobExpansionConjunction, PathGlobs, PreparedPathGlobs, StrictGlobMatch
 use hashing::{Digest, Fingerprint};
 use store::Snapshot;
 
-pub(crate) fn register(m: &PyModule) -> PyResult<()> {
+pub fn register(m: &PyModule) -> PyResult<()> {
   m.add_function(wrap_pyfunction!(match_path_globs, m)?)?;
   m.add_function(wrap_pyfunction!(default_cache_path, m)?)?;
   m.add_class::<PyDigest>()?;

src/rust/engine/engine_pyo3/src/lib.rs

@@ -32,4 +32,8 @@
 #![allow(clippy::not_unsafe_ptr_arg_deref)]
 #![type_length_limit = "43757804"]
 
-mod externs;
+pub mod fs;
+pub mod nailgun;
+pub mod scheduler;
+pub mod testutil;
+pub mod workunits;

src/rust/engine/engine_pyo3/src/externs/interface/nailgun.rs → src/rust/engine/engine_pyo3/src/nailgun.rs

@@ -1,14 +1,15 @@
 // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-use super::PyExecutor;
 use pyo3::create_exception;
 use pyo3::exceptions::{PyBrokenPipeError, PyException, PyKeyboardInterrupt};
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
+
+use crate::scheduler::PyExecutor;
 use task_executor::Executor;
 
-pub(crate) fn register(py: Python, m: &PyModule) -> PyResult<()> {
+pub fn register(py: Python, m: &PyModule) -> PyResult<()> {
   m.add(
     "PantsdConnectionException",
     py.get_type::<PantsdConnectionException>(),

src/rust/engine/engine_pyo3/src/externs/interface.rs → src/rust/engine/engine_pyo3/src/scheduler.rs

@@ -4,26 +4,14 @@
 use pyo3::exceptions::PyException;
 use pyo3::prelude::*;
 
-mod fs;
-mod nailgun;
-mod testutil;
-mod workunits;
-
-#[pymodule]
-fn native_engine_pyo3(py: Python, m: &PyModule) -> PyResult<()> {
-  self::fs::register(m)?;
-  self::nailgun::register(py, m)?;
-  self::testutil::register(m)?;
-  self::workunits::register(m)?;
-
+pub fn register(m: &PyModule) -> PyResult<()> {
   m.add_class::<PyExecutor>()?;
-
   Ok(())
 }
 
 #[pyclass]
 #[derive(Debug, Clone)]
-struct PyExecutor(task_executor::Executor);
+pub struct PyExecutor(pub task_executor::Executor);
 
 #[pymethods]
 impl PyExecutor {

src/rust/engine/engine_pyo3/src/externs/interface/testutil.rs → src/rust/engine/engine_pyo3/src/testutil.rs

@@ -3,14 +3,15 @@
 
 use std::sync::Arc;
 
-use super::PyExecutor;
 use parking_lot::Mutex;
 use pyo3::exceptions::PyAssertionError;
 use pyo3::prelude::*;
 use pyo3::types::PyType;
+
+use crate::scheduler::PyExecutor;
 use testutil_mock::{StubCAS, StubCASBuilder};
 
-pub(crate) fn register(m: &PyModule) -> PyResult<()> {
+pub fn register(m: &PyModule) -> PyResult<()> {
   m.add_class::<PyStubCAS>()?;
   m.add_class::<PyStubCASBuilder>()?;
   Ok(())

src/rust/engine/engine_pyo3/src/externs/interface/workunits.rs → src/rust/engine/engine_pyo3/src/workunits.rs

@@ -2,9 +2,10 @@
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 use pyo3::prelude::*;
+
 use workunit_store::Metric;
 
-pub(crate) fn register(m: &PyModule) -> PyResult<()> {
+pub fn register(m: &PyModule) -> PyResult<()> {
   m.add_function(wrap_pyfunction!(all_counter_names, m)?)?;
   Ok(())
 }

src/rust/engine/engine_pyo3_extension/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+version = "0.0.1"
+edition = "2018"
+name = "engine_pyo3_extension"
+authors = [ "Pants Build <[email protected]>" ]
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[features]
+# NB: To actually load this crate from python, the `extension-module` feature must be enabled. But
+# unfortunately, enabling `extension-module` causes tests linked against `pyo3` to fail. We
+# define a feature to enable that, but we do not enable it by default: someone building this module
+# in order to extract `libengine_pyo3.so` should pass `cargo build .. --features=engine_pyo3/extension-module`.
+#  see https://github.com/PyO3/pyo3/issues/340
+extension-module = ["pyo3/extension-module"]
+default = []
+
+[dependencies]
+engine_pyo3 = { "path" = "../engine_pyo3" }
+pyo3 = "0.14.1"

src/rust/engine/engine_pyo3_extension/build.rs

@@ -0,0 +1,34 @@
+// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+#![deny(warnings)]
+// Enable all clippy lints except for many of the pedantic ones. It's a shame this needs to be copied and pasted across crates, but there doesn't appear to be a way to include inner attributes from a common source.
+#![deny(
+  clippy::all,
+  clippy::default_trait_access,
+  clippy::expl_impl_clone_on_copy,
+  clippy::if_not_else,
+  clippy::needless_continue,
+  clippy::unseparated_literal_suffix,
+  // TODO: Falsely triggers for async/await:
+  //   see https://github.com/rust-lang/rust-clippy/issues/5360
+  // clippy::used_underscore_binding
+)]
+// It is often more clear to show that nothing is being moved.
+#![allow(clippy::match_ref_pats)]
+// Subjective style.
+#![allow(
+  clippy::len_without_is_empty,
+  clippy::redundant_field_names,
+  clippy::too_many_arguments
+)]
+// Default isn't as big a deal as people seem to think it is.
+#![allow(clippy::new_without_default, clippy::new_ret_no_self)]
+// Arc<Mutex> can be more clear than needing to grok Orderings:
+#![allow(clippy::mutex_atomic)]
+
+fn main() {
+  // NB: The native extension only works with the Python interpreter version it was built with
+  // (e.g. Python 3.7 vs 3.8).
+  println!("cargo:rerun-if-env-changed=PY");
+}

src/rust/engine/engine_pyo3_extension/src/lib.rs

@@ -0,0 +1,40 @@
+// Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+#![deny(warnings)]
+// Enable all clippy lints except for many of the pedantic ones. It's a shame this needs to be copied and pasted across crates, but there doesn't appear to be a way to include inner attributes from a common source.
+#![deny(
+  clippy::all,
+  clippy::default_trait_access,
+  clippy::expl_impl_clone_on_copy,
+  clippy::if_not_else,
+  clippy::needless_continue,
+  clippy::unseparated_literal_suffix,
+  // TODO: Falsely triggers for async/await:
+  //   see https://github.com/rust-lang/rust-clippy/issues/5360
+  // clippy::used_underscore_binding
+)]
+// It is often more clear to show that nothing is being moved.
+#![allow(clippy::match_ref_pats)]
+// Subjective style.
+#![allow(
+  clippy::len_without_is_empty,
+  clippy::redundant_field_names,
+  clippy::too_many_arguments
+)]
+// Default isn't as big a deal as people seem to think it is.
+#![allow(clippy::new_without_default, clippy::new_ret_no_self)]
+// Arc<Mutex> can be more clear than needing to grok Orderings:
+#![allow(clippy::mutex_atomic)]
+
+use pyo3::prelude::*;
+
+#[pymodule]
+fn native_engine_pyo3(py: Python, m: &PyModule) -> PyResult<()> {
+  engine_pyo3::fs::register(m)?;
+  engine_pyo3::nailgun::register(py, m)?;
+  engine_pyo3::scheduler::register(m)?;
+  engine_pyo3::testutil::register(m)?;
+  engine_pyo3::workunits::register(m)?;
+  Ok(())
+}

src/rust/engine/src/externs/mod.rs

@@ -28,6 +28,7 @@ use cpython::{
   PyObject, PyResult as CPyResult, PyTuple, PyType, Python, PythonObject, ToPyObject,
 };
 use lazy_static::lazy_static;
+use pyo3::FromPyPointer;
 
 use logging::PythonLogLevel;
 
@@ -420,3 +421,10 @@ pub enum GeneratorResponse {
   Get(Get),
   GetMulti(Vec<Get>),
 }
+
+pub(crate) fn from_rust_cpython_to_pyo3<'p>(
+  value: &PyObject,
+  py: pyo3::Python<'p>,
+) -> &'p pyo3::PyAny {
+  unsafe { pyo3::PyAny::from_owned_ptr(py, value.as_ptr() as *mut pyo3::ffi::PyObject) }
+}