From cd1c9d4b525b4faea025219375e61bb88d78ac69 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 22 Feb 2025 11:25:12 -0800 Subject: [PATCH 01/25] Pure `git merge --squash smart_holder` (no manual interventions). --- .codespell-ignore-lines | 11 + .github/workflows/ci.yml | 15 + .github/workflows/configure.yml | 1 + .github/workflows/emscripten.yaml | 1 + .github/workflows/format.yml | 1 + .github/workflows/pip.yml | 1 + .pre-commit-config.yaml | 2 + CMakeLists.txt | 5 + MANIFEST.in | 2 + README.rst | 4 + README_smart_holder.rst | 123 ++++++ docs/advanced/smart_ptrs.rst | 7 + include/pybind11/attr.h | 18 +- include/pybind11/cast.h | 283 +++++++++++- include/pybind11/detail/class.h | 2 + include/pybind11/detail/common.h | 2 + .../detail/dynamic_raw_ptr_cast_if_possible.h | 39 ++ include/pybind11/detail/init.h | 71 ++- include/pybind11/detail/internals.h | 17 +- include/pybind11/detail/struct_smart_holder.h | 349 +++++++++++++++ include/pybind11/detail/type_caster_base.h | 368 ++++++++++++++++ include/pybind11/detail/using_smart_holder.h | 22 + include/pybind11/detail/value_and_holder.h | 1 + include/pybind11/pybind11.h | 334 +++++++++++++- include/pybind11/smart_holder.h | 14 + include/pybind11/stl_bind.h | 4 +- .../pybind11/trampoline_self_life_support.h | 60 +++ tests/CMakeLists.txt | 20 + tests/extra_python_package/test_files.py | 6 + tests/pure_cpp/CMakeLists.txt | 20 + tests/pure_cpp/smart_holder_poc.h | 51 +++ tests/pure_cpp/smart_holder_poc_test.cpp | 415 ++++++++++++++++++ tests/test_class.cpp | 13 +- tests/test_class.py | 10 + tests/test_class_sh_basic.cpp | 249 +++++++++++ tests/test_class_sh_basic.py | 246 +++++++++++ tests/test_class_sh_disowning.cpp | 43 ++ tests/test_class_sh_disowning.py | 78 ++++ tests/test_class_sh_disowning_mi.cpp | 87 ++++ tests/test_class_sh_disowning_mi.py | 246 +++++++++++ tests/test_class_sh_factory_constructors.cpp | 166 +++++++ tests/test_class_sh_factory_constructors.py | 53 +++ tests/test_class_sh_inheritance.cpp | 92 ++++ tests/test_class_sh_inheritance.py | 63 +++ tests/test_class_sh_mi_thunks.cpp | 96 ++++ tests/test_class_sh_mi_thunks.py | 53 +++ tests/test_class_sh_property.cpp | 95 ++++ tests/test_class_sh_property.py | 166 +++++++ tests/test_class_sh_property_non_owning.cpp | 64 +++ tests/test_class_sh_property_non_owning.py | 30 ++ tests/test_class_sh_shared_ptr_copy_move.cpp | 105 +++++ tests/test_class_sh_shared_ptr_copy_move.py | 41 ++ tests/test_class_sh_trampoline_basic.cpp | 84 ++++ tests/test_class_sh_trampoline_basic.py | 59 +++ ..._class_sh_trampoline_self_life_support.cpp | 87 ++++ ...t_class_sh_trampoline_self_life_support.py | 38 ++ ...t_class_sh_trampoline_shared_from_this.cpp | 138 ++++++ ...st_class_sh_trampoline_shared_from_this.py | 247 +++++++++++ ...class_sh_trampoline_shared_ptr_cpp_arg.cpp | 93 ++++ ..._class_sh_trampoline_shared_ptr_cpp_arg.py | 154 +++++++ tests/test_class_sh_trampoline_unique_ptr.cpp | 64 +++ tests/test_class_sh_trampoline_unique_ptr.py | 31 ++ ...est_class_sh_unique_ptr_custom_deleter.cpp | 32 ++ ...test_class_sh_unique_ptr_custom_deleter.py | 8 + tests/test_class_sh_unique_ptr_member.cpp | 52 +++ tests/test_class_sh_unique_ptr_member.py | 26 ++ tests/test_class_sh_virtual_py_cpp_mix.cpp | 60 +++ tests/test_class_sh_virtual_py_cpp_mix.py | 66 +++ ubench/holder_comparison.cpp | 49 +++ ubench/holder_comparison.py | 144 ++++++ .../holder_comparison_extract_sheet_data.py | 71 +++ ubench/number_bucket.h | 55 +++ ubench/python/number_bucket.clif | 6 + 73 files changed, 5801 insertions(+), 28 deletions(-) create mode 100644 README_smart_holder.rst create mode 100644 include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h create mode 100644 include/pybind11/detail/struct_smart_holder.h create mode 100644 include/pybind11/detail/using_smart_holder.h create mode 100644 include/pybind11/smart_holder.h create mode 100644 include/pybind11/trampoline_self_life_support.h create mode 100644 tests/pure_cpp/CMakeLists.txt create mode 100644 tests/pure_cpp/smart_holder_poc.h create mode 100644 tests/pure_cpp/smart_holder_poc_test.cpp create mode 100644 tests/test_class_sh_basic.cpp create mode 100644 tests/test_class_sh_basic.py create mode 100644 tests/test_class_sh_disowning.cpp create mode 100644 tests/test_class_sh_disowning.py create mode 100644 tests/test_class_sh_disowning_mi.cpp create mode 100644 tests/test_class_sh_disowning_mi.py create mode 100644 tests/test_class_sh_factory_constructors.cpp create mode 100644 tests/test_class_sh_factory_constructors.py create mode 100644 tests/test_class_sh_inheritance.cpp create mode 100644 tests/test_class_sh_inheritance.py create mode 100644 tests/test_class_sh_mi_thunks.cpp create mode 100644 tests/test_class_sh_mi_thunks.py create mode 100644 tests/test_class_sh_property.cpp create mode 100644 tests/test_class_sh_property.py create mode 100644 tests/test_class_sh_property_non_owning.cpp create mode 100644 tests/test_class_sh_property_non_owning.py create mode 100644 tests/test_class_sh_shared_ptr_copy_move.cpp create mode 100644 tests/test_class_sh_shared_ptr_copy_move.py create mode 100644 tests/test_class_sh_trampoline_basic.cpp create mode 100644 tests/test_class_sh_trampoline_basic.py create mode 100644 tests/test_class_sh_trampoline_self_life_support.cpp create mode 100644 tests/test_class_sh_trampoline_self_life_support.py create mode 100644 tests/test_class_sh_trampoline_shared_from_this.cpp create mode 100644 tests/test_class_sh_trampoline_shared_from_this.py create mode 100644 tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp create mode 100644 tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py create mode 100644 tests/test_class_sh_trampoline_unique_ptr.cpp create mode 100644 tests/test_class_sh_trampoline_unique_ptr.py create mode 100644 tests/test_class_sh_unique_ptr_custom_deleter.cpp create mode 100644 tests/test_class_sh_unique_ptr_custom_deleter.py create mode 100644 tests/test_class_sh_unique_ptr_member.cpp create mode 100644 tests/test_class_sh_unique_ptr_member.py create mode 100644 tests/test_class_sh_virtual_py_cpp_mix.cpp create mode 100644 tests/test_class_sh_virtual_py_cpp_mix.py create mode 100644 ubench/holder_comparison.cpp create mode 100644 ubench/holder_comparison.py create mode 100644 ubench/holder_comparison_extract_sheet_data.py create mode 100644 ubench/number_bucket.h create mode 100644 ubench/python/number_bucket.clif diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines index 2a01d63ebb..e8cbf31447 100644 --- a/.codespell-ignore-lines +++ b/.codespell-ignore-lines @@ -12,6 +12,17 @@ template template class_ &def(const detail::op_ &op, const Extra &...extra) { class_ &def_cast(const detail::op_ &op, const Extra &...extra) { + int valu; + explicit movable_int(int v) : valu{v} {} + movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; } + explicit indestructible_int(int v) : valu{v} {} + REQUIRE(hld.as_raw_ptr_unowned()->valu == 19); + REQUIRE(othr.valu == 19); + REQUIRE(orig.valu == 91); + (m.pass_valu, "Valu", "pass_valu:Valu(_MvCtor)*_CpCtor"), +atyp_valu rtrn_valu() { atyp_valu obj{"Valu"}; return obj; } + assert m.atyp_valu().get_mtxt() == "Valu" +// valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const, @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) struct IntStruct { explicit IntStruct(int v) : value(v){}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e281db73..d8607ee988 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: branches: - master - stable + - smart_holder - v* permissions: read-all @@ -87,6 +88,20 @@ jobs: # Extra ubuntu latest job - runs-on: ubuntu-latest python: '3.11' + # Exercise PYBIND11_USE_SMART_HOLDER_AS_DEFAULT + # with recent (or ideally latest) released Python version. + - runs-on: ubuntu-latest + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT" + - runs-on: macos-13 + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT" + - runs-on: windows-2022 + python: '3.12' + args: > + -DCMAKE_CXX_FLAGS="/DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT /GR /EHsc" exclude: # The setup-python action currently doesn't have graalpy for windows - python: 'graalpy-24.1' diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 2031ec8236..8e89e6f394 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -7,6 +7,7 @@ on: branches: - master - stable + - smart_holder - v* permissions: diff --git a/.github/workflows/emscripten.yaml b/.github/workflows/emscripten.yaml index 5eac089e5c..c7fd73cdf3 100644 --- a/.github/workflows/emscripten.yaml +++ b/.github/workflows/emscripten.yaml @@ -6,6 +6,7 @@ on: branches: - master - stable + - smart_holder - v* concurrency: diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e50dc0bb72..6bb22171b7 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -10,6 +10,7 @@ on: branches: - master - stable + - smart_holder - "v*" permissions: diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c10087eff4..a32e7fe8f1 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -7,6 +7,7 @@ on: branches: - master - stable + - smart_holder - v* release: types: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1abc48072a..fbecbc0aef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,6 +76,7 @@ repos: - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace + exclude: \.patch?$ # Also code format the docs - repo: https://github.com/adamchainz/blacken-docs @@ -90,6 +91,7 @@ repos: rev: "v1.5.5" hooks: - id: remove-tabs + exclude: (^docs/.*|\.patch)?$ # Avoid directional quotes - repo: https://github.com/sirosen/texthooks diff --git a/CMakeLists.txt b/CMakeLists.txt index 9357f31fbd..d8323c10f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,10 +131,13 @@ set(PYBIND11_HEADERS include/pybind11/detail/common.h include/pybind11/detail/cpp_conduit.h include/pybind11/detail/descr.h + include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h include/pybind11/detail/init.h include/pybind11/detail/internals.h + include/pybind11/detail/struct_smart_holder.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h + include/pybind11/detail/using_smart_holder.h include/pybind11/detail/value_and_holder.h include/pybind11/detail/exception_translation.h include/pybind11/attr.h @@ -161,9 +164,11 @@ set(PYBIND11_HEADERS include/pybind11/operators.h include/pybind11/pybind11.h include/pybind11/pytypes.h + include/pybind11/smart_holder.h include/pybind11/stl.h include/pybind11/stl_bind.h include/pybind11/stl/filesystem.h + include/pybind11/trampoline_self_life_support.h include/pybind11/type_caster_pyobject_ptr.h include/pybind11/typing.h include/pybind11/warnings.h) diff --git a/MANIFEST.in b/MANIFEST.in index 7ce83c5527..ff9ee6c74f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ prune tests +prune ubench +include README_smart_holder.rst recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed diff --git a/README.rst b/README.rst index 8e0c391770..54da6349f9 100644 --- a/README.rst +++ b/README.rst @@ -13,6 +13,10 @@ .. start +.. Note:: + + This is the pybind11 **smart_holder** branch. Please refer to + ``README_smart_holder.rst`` for branch-specific information. **pybind11** is a lightweight header-only library that exposes C++ types in Python and vice versa, mainly to create Python bindings of existing diff --git a/README_smart_holder.rst b/README_smart_holder.rst new file mode 100644 index 0000000000..05b1ab56fa --- /dev/null +++ b/README_smart_holder.rst @@ -0,0 +1,123 @@ +============================== +pybind11 — smart_holder branch +============================== + + +Overview +======== + +- The smart_holder branch is a strict superset of the pybind11 master branch. + Everything that works with the master branch is expected to work exactly the + same with the smart_holder branch. + +- Activating the smart_holder functionality for a given C++ type ``T`` is as + easy as changing ``py::class_`` to ``py::classh`` in client code. + +- The ``py::classh`` functionality includes + + * support for **two-way** Python/C++ conversions for both + ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. + — In contrast, ``py::class_`` only supports one-way C++-to-Python + conversions for ``std::unique_ptr``, or alternatively two-way + Python/C++ conversions for ``std::shared_ptr``, which then excludes + the one-way C++-to-Python ``std::unique_ptr`` conversions (this + manifests itself through undefined runtime behavior). + + * passing a Python object back to C++ via ``std::unique_ptr``, safely + **disowning** the Python object. + + * safely passing `"trampoline" + `_ + objects (objects with C++ virtual function overrides implemented in + Python) via ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: + associated Python objects are automatically kept alive for the lifetime + of the smart-pointer. + +Note: As of `PR #5257 `_ +the smart_holder functionality is fully baked into pybind11. +Prior to PR #5257 the smart_holder implementation was an "add-on", which made +it necessary to use a ``PYBIND11_SMART_HOLDER_TYPE_CASTERS`` macro. This macro +still exists for backward compatibility, but is now a no-op. The trade-off +for this convenience is that the ``PYBIND11_INTERNALS_VERSION`` needed to be +changed. Consequently, Python extension modules built with the smart_holder +branch no longer interoperate with extension modules built with the pybind11 +master branch. If cross-extension-module interoperability is required, all +extension modules involved must be built with the smart_holder branch. +— Probably, most extension modules do not require cross-extension-module +interoperability, but exceptions to this are quite common. + + +What is fundamentally different? +-------------------------------- + +- Classic pybind11 has the concept of "smart-pointer is holder". + Interoperability between smart-pointers is completely missing. For example, + with ``py::class_>``, ``return``-ing a + ``std::unique_ptr`` leads to undefined runtime behavior + (`#1138 `_). + A `systematic analysis can be found here + `_. + +- ``py::smart_holder`` has a richer concept in comparison, with well-defined + runtime behavior in all situations. ``py::smart_holder`` "knows" about both + ``std::unique_ptr`` and ``std::shared_ptr``, and how they interoperate. + + +What motivated the development of the smart_holder code? +-------------------------------------------------------- + +- The original context was retooling of `PyCLIF + `_, to use pybind11 underneath, + instead of directly targeting the Python C API. Essentially the smart_holder + branch is porting established PyCLIF functionality into pybind11. (However, + this work also led to bug fixes in PyCLIF.) + + +Installation +============ + +Currently ``git clone`` is the only option. We do not have released packages. + +.. code-block:: bash + + git clone --branch smart_holder https://github.com/pybind/pybind11.git + +Everything else is exactly identical to using the default (master) branch. + + +Trampolines and std::unique_ptr +------------------------------- + +A pybind11 `"trampoline" +`_ +is a C++ helper class with virtual function overrides that transparently +call back from C++ into Python. To enable safely passing a ``std::unique_ptr`` +to a trampoline object between Python and C++, the trampoline class must +inherit from ``py::trampoline_self_life_support``, for example: + +.. code-block:: cpp + + class PyAnimal : public Animal, public py::trampoline_self_life_support { + ... + }; + +This is the only difference compared to classic pybind11. A fairly +minimal but complete example is tests/test_class_sh_trampoline_unique_ptr.cpp. + + +Related links +============= + +* The smart_holder branch addresses issue + `#1138 `_ and + the ten issues enumerated in the `description of PR 2839 + `_. + +* `Description of PR #2672 + `_, from which + the smart_holder branch was created. + +* Small `slide deck + `_ + presented in meeting with pybind11 maintainers on Feb 22, 2021. Slides 5 + and 6 show performance comparisons. (These are outdated but probably not far off.) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index b9f100cf8e..e2e9f3f37f 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -1,6 +1,13 @@ Smart pointers ############## +.. Note:: + + This is the pybind11 **smart_holder** branch, but the information + below has NOT been updated accordingly yet. Please refer to + ``README_smart_holder.rst`` under the top-level pybind11 directory + for updated information about smart pointers. + std::unique_ptr =============== diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index eb15ffd073..ef2ca1709c 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -276,8 +276,7 @@ struct function_record { struct type_record { PYBIND11_NOINLINE type_record() : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), - default_holder(true), module_local(false), is_final(false), - release_gil_before_calling_cpp_dtor(false) {} + module_local(false), is_final(false), release_gil_before_calling_cpp_dtor(false) {} /// Handle to the parent scope handle scope; @@ -327,9 +326,6 @@ struct type_record { /// Does the class implement the buffer protocol? bool buffer_protocol : 1; - /// Is the default (unique_ptr) holder type used? - bool default_holder : 1; - /// Is the class definition local to the module shared object? bool module_local : 1; @@ -339,6 +335,8 @@ struct type_record { /// Solves pybind/pybind11#1446 bool release_gil_before_calling_cpp_dtor : 1; + holder_enum_t holder_enum_v = holder_enum_t::undefined; + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { auto *base_info = detail::get_type_info(base, false); if (!base_info) { @@ -348,13 +346,17 @@ struct type_record { + "\" referenced unknown base type \"" + tname + "\""); } - if (default_holder != base_info->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool this_has_unique_ptr_holder = (holder_enum_v == holder_enum_t::std_unique_ptr); + bool base_has_unique_ptr_holder + = (base_info->holder_enum_v == holder_enum_t::std_unique_ptr); + if (this_has_unique_ptr_holder != base_has_unique_ptr_holder) { std::string tname(base.name()); detail::clean_type_id(tname); pybind11_fail("generic_type: type \"" + std::string(name) + "\" " - + (default_holder ? "does not have" : "has") + + (this_has_unique_ptr_holder ? "does not have" : "has") + " a non-default holder type while its base \"" + tname + "\" " - + (base_info->default_holder ? "does not" : "does")); + + (base_has_unique_ptr_holder ? "does not" : "does")); } bases.append((PyObject *) base_info->type); diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index f2c029113a..13a589f179 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -754,6 +754,7 @@ struct holder_helper { static auto get(const T &p) -> decltype(p.get()) { return p.get(); } }; +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization. /// Type caster for holder types like std::shared_ptr, etc. /// The SFINAE hook is provided to help work around the current lack of support /// for smart-pointer interoperability. Please consider it an implementation @@ -789,7 +790,10 @@ struct copyable_holder_caster : public type_caster_base { protected: friend class type_caster_generic; void check_holder_compat() { - if (typeinfo->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { throw cast_error("Unable to load a custom holder type from a default-holder instance"); } } @@ -835,10 +839,144 @@ struct copyable_holder_caster : public type_caster_base { holder_type holder; }; +template +struct copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor copyable_holder_caster to reduce code duplication. +template +struct copyable_holder_caster< + type, + std::shared_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + bool load(handle src, bool convert) { + if (base::template load_impl>>( + src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + explicit operator std::shared_ptr *() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + pybind11_fail("Passing `std::shared_ptr *` from Python to C++ is not supported " + "(inherently unsafe)."); + } + return std::addressof(shared_ptr_storage); + } + + explicit operator std::shared_ptr &() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value); + } + return shared_ptr_storage; + } + + static handle + cast(const std::shared_ptr &src, return_value_policy policy, handle parent) { + const auto *ptr = src.get(); + auto st = type_caster_base::src_and_type(ptr); + if (st.second == nullptr) { + return handle(); // no type info: error will be set already + } + if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + return smart_holder_type_caster_support::smart_holder_from_shared_ptr( + src, policy, parent, st); + } + return type_caster_base::cast_holder(ptr, &src); + } + + // This function will succeed even if the `responsible_parent` does not own the + // wrapped C++ object directly. + // It is the responsibility of the caller to ensure that the `responsible_parent` + // has a `keep_alive` relationship with the owner of the wrapped C++ object, or + // that the wrapped C++ object lives for the duration of the process. + static std::shared_ptr shared_ptr_with_responsible_parent(handle responsible_parent) { + copyable_holder_caster loader; + loader.load(responsible_parent, /*convert=*/false); + assert(loader.typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder); + return loader.sh_load_helper.load_as_shared_ptr(loader.value, responsible_parent); + } + +protected: + friend class type_caster_generic; + void check_holder_compat() { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { + throw cast_error("Unable to load a custom holder type from a default-holder instance"); + } + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + if (v_h.holder_constructed()) { + value = v_h.value_ptr(); + shared_ptr_storage = v_h.template holder>(); + return; + } + throw cast_error("Unable to cast from non-held to held instance (T& to Holder) " +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "type information)"); +#else + "of type '" + + type_id>() + "''"); +#endif + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle, bool) { + return false; + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + copyable_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + } else { + shared_ptr_storage + = std::shared_ptr(sub_caster.shared_ptr_storage, (type *) value); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; +}; + /// Specialize for the common std::shared_ptr, so users don't need to template class type_caster> : public copyable_holder_caster> {}; +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to unique_ptr specialization. /// Type caster for holder types like std::unique_ptr. /// Please consider the SFINAE hook an implementation detail, as explained /// in the comment for the copyable_holder_caster. @@ -854,6 +992,143 @@ struct move_only_holder_caster { static constexpr auto name = type_caster_base::name; }; +template +struct move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor move_only_holder_caster to reduce code duplication. +template +struct move_only_holder_caster< + type, + std::unique_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + static handle + cast(std::unique_ptr &&src, return_value_policy policy, handle parent) { + auto *ptr = src.get(); + auto st = type_caster_base::src_and_type(ptr); + if (st.second == nullptr) { + return handle(); // no type info: error will be set already + } + if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) { + return smart_holder_type_caster_support::smart_holder_from_unique_ptr( + std::move(src), policy, parent, st); + } + return type_caster_generic::cast(st.first, + return_value_policy::take_ownership, + {}, + st.second, + nullptr, + nullptr, + std::addressof(src)); + } + + static handle + cast(const std::unique_ptr &src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } + if (policy == return_value_policy::automatic) { + policy = return_value_policy::reference_internal; + } + if (policy != return_value_policy::reference_internal) { + throw cast_error("Invalid return_value_policy for const unique_ptr&"); + } + return type_caster_base::cast(src.get(), policy, parent); + } + + bool load(handle src, bool convert) { + if (base::template load_impl< + move_only_holder_caster>>(src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.loaded_v_h.type = typeinfo; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + pybind11_fail( + "Passing `std::unique_ptr` from Python to C++ requires `py::classh` (with T = " + + clean_type_id(typeinfo->cpptype->name()) + ")"); + } + + template + using cast_op_type + = conditional_t::type, + const std::unique_ptr &>::value + || std::is_same::type, + const std::unique_ptr &>::value, + const std::unique_ptr &, + std::unique_ptr>; + + explicit operator std::unique_ptr() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + return sh_load_helper.template load_as_unique_ptr(value); + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + explicit operator const std::unique_ptr &() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + // Get shared_ptr to ensure that the Python object is not disowned elsewhere. + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(value); + // Build a temporary unique_ptr that is meant to never expire. + unique_ptr_storage = std::shared_ptr>( + new std::unique_ptr{ + sh_load_helper.template load_as_const_unique_ptr( + shared_ptr_storage.get())}, + [](std::unique_ptr *ptr) { + if (!ptr) { + pybind11_fail("FATAL: `const std::unique_ptr &` was disowned " + "(EXPECT UNDEFINED BEHAVIOR)."); + } + (void) ptr->release(); + delete ptr; + }); + return *unique_ptr_storage; + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + move_only_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + } else { + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ + ":" PYBIND11_TOSTRING(__LINE__)); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; // Serves as a pseudo lock. + std::shared_ptr> unique_ptr_storage; +}; + template class type_caster> : public move_only_holder_caster> {}; @@ -887,10 +1162,14 @@ struct always_construct_holder : always_construct_holder_value {}; template struct is_holder_type : std::is_base_of, detail::type_caster> {}; -// Specialization for always-supported unique_ptr holders: + +// Specializations for always-supported holders: template struct is_holder_type> : std::true_type {}; +template +struct is_holder_type : std::true_type {}; + #ifdef PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION // See PR #4888 // This leads to compilation errors if a specialization is missing. diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index aac6e847c6..08e23afb59 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -457,6 +457,8 @@ inline void clear_instance(PyObject *self) { if (instance->owned || v_h.holder_constructed()) { v_h.type->dealloc(v_h); } + } else if (v_h.holder_constructed()) { + v_h.type->dealloc(v_h); // Disowned instance. } } // Deallocate the value/holder layout internals: diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 372dd069a1..5e225f8c19 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -605,6 +605,8 @@ struct instance { bool simple_instance_registered : 1; /// If true, get_internals().patients has an entry for this object bool has_patients : 1; + /// If true, this Python object needs to be kept alive for the lifetime of the C++ value. + bool is_alias : 1; /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) diff --git a/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h new file mode 100644 index 0000000000..7c00fe98c1 --- /dev/null +++ b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct dynamic_raw_ptr_cast_is_possible : std::false_type {}; + +template +struct dynamic_raw_ptr_cast_is_possible< + To, + From, + detail::enable_if_t::value && std::is_polymorphic::value>> + : std::true_type {}; + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From * /*ptr*/) { + return nullptr; +} + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From *ptr) { + return dynamic_cast(ptr); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 3eeeaaf970..ed95afe58c 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -10,6 +10,7 @@ #pragma once #include "class.h" +#include "using_smart_holder.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -155,7 +156,7 @@ void construct(value_and_holder &v_h, Alias *alias_ptr, bool) { // holder. This also handles types like std::shared_ptr and std::unique_ptr where T is a // derived type (through those holder's implicit conversion from derived class holder // constructors). -template +template >::value, int> = 0> void construct(value_and_holder &v_h, Holder holder, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); auto *ptr = holder_helper>::get(holder); @@ -197,6 +198,74 @@ void construct(value_and_holder &v_h, Alias &&result, bool) { v_h.value_ptr() = new Alias(std::move(result)); } +template +smart_holder init_smart_holder_from_unique_ptr(std::unique_ptr &&unq_ptr, + bool void_cast_raw_ptr) { + void *void_ptr = void_cast_raw_ptr ? static_cast(unq_ptr.get()) : nullptr; + return smart_holder::from_unique_ptr(std::move(unq_ptr), void_ptr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, std::unique_ptr, D> &&unq_ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::unique_ptr pointee " + "is not an alias instance"); + } + // Here and below: if the new object is a trampoline, the shared_from_this mechanism needs + // to be prevented from accessing the smart_holder vptr, because it does not keep the + // trampoline Python object alive. For types that don't inherit from enable_shared_from_this + // it does not matter if void_cast_raw_ptr is true or false, therefore it's not necessary + // to also inspect the type. + auto smhldr = init_smart_holder_from_unique_ptr( + std::move(unq_ptr), /*void_cast_raw_ptr*/ Class::has_alias && is_alias(ptr)); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, + std::unique_ptr, D> &&unq_ptr, + bool /*need_alias*/) { + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + auto smhldr + = init_smart_holder_from_unique_ptr(std::move(unq_ptr), /*void_cast_raw_ptr*/ true); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, std::shared_ptr> &&shd_ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::shared_ptr pointee " + "is not an alias instance"); + } + auto smhldr = smart_holder::from_shared_ptr(shd_ptr); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, + std::shared_ptr> &&shd_ptr, + bool /*need_alias*/) { + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + auto smhldr = smart_holder::from_shared_ptr(shd_ptr); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + // Implementing class for py::init<...>() template struct constructor { diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index dffb9e444f..234524f910 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -37,11 +37,11 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 6 +# define PYBIND11_INTERNALS_VERSION 7 #endif -#if PYBIND11_INTERNALS_VERSION < 6 -# error "PYBIND11_INTERNALS_VERSION 6 is the minimum for all platforms for pybind11v3." +#if PYBIND11_INTERNALS_VERSION < 7 +# error "PYBIND11_INTERNALS_VERSION 7 is the minimum for all platforms for pybind11v3." #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -211,6 +211,14 @@ struct internals { } }; +enum class holder_enum_t : uint8_t { + undefined, + std_unique_ptr, // Default, lacking interop with std::shared_ptr. + std_shared_ptr, // Lacking interop with std::unique_ptr. + smart_holder, // Full std::unique_ptr / std::shared_ptr interop. + custom_holder, +}; + /// Additional type information which does not fit into the PyTypeObject. /// Changes to this struct also require bumping `PYBIND11_INTERNALS_VERSION`. struct type_info { @@ -226,6 +234,7 @@ struct type_info { buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; void *get_buffer_data = nullptr; void *(*module_local_load)(PyObject *, const type_info *) = nullptr; + holder_enum_t holder_enum_v = holder_enum_t::undefined; /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. @@ -233,8 +242,6 @@ struct type_info { bool simple_type : 1; /* True if there is no multiple inheritance in this type's inheritance tree */ bool simple_ancestors : 1; - /* for base vs derived holder_type checks */ - bool default_holder : 1; /* true if this is a type registered with py::module_local */ bool module_local : 1; }; diff --git a/include/pybind11/detail/struct_smart_holder.h b/include/pybind11/detail/struct_smart_holder.h new file mode 100644 index 0000000000..980fc3699b --- /dev/null +++ b/include/pybind11/detail/struct_smart_holder.h @@ -0,0 +1,349 @@ +// Copyright (c) 2020-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/* Proof-of-Concept for smart pointer interoperability. + +High-level aspects: + +* Support all `unique_ptr`, `shared_ptr` interops that are feasible. + +* Cleanly and clearly report all interops that are infeasible. + +* Meant to fit into a `PyObject`, as a holder for C++ objects. + +* Support a system design that makes it impossible to trigger + C++ Undefined Behavior, especially from Python. + +* Support a system design with clean runtime inheritance casting. From this + it follows that the `smart_holder` needs to be type-erased (`void*`). + +* Handling of RTTI for the type-erased held pointer is NOT implemented here. + It is the responsibility of the caller to ensure that `static_cast` + is well-formed when calling `as_*` member functions. Inheritance casting + needs to be handled in a different layer (similar to the code organization + in boost/python/object/inheritance.hpp). + +Details: + +* The "root holder" chosen here is a `shared_ptr` (named `vptr` in this + implementation). This choice is practically inevitable because `shared_ptr` + has only very limited support for inspecting and accessing its deleter. + +* If created from a raw pointer, or a `unique_ptr` without a custom deleter, + `vptr` always uses a custom deleter, to support `unique_ptr`-like disowning. + The custom deleters could be extended to included life-time management for + external objects (e.g. `PyObject`). + +* If created from an external `shared_ptr`, or a `unique_ptr` with a custom + deleter, including life-time management for external objects is infeasible. + +* By choice, the smart_holder is movable but not copyable, to keep the design + simple, and to guard against accidental copying overhead. + +* The `void_cast_raw_ptr` option is needed to make the `smart_holder` `vptr` + member invisible to the `shared_from_this` mechanism, in case the lifetime + of a `PyObject` is tied to the pointee. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// pybindit = Python Bindings Innovation Track. +// Currently not in pybind11 namespace to signal that this POC does not depend +// on any existing pybind11 functionality. +namespace pybindit { +namespace memory { + +static constexpr bool type_has_shared_from_this(...) { return false; } + +template +static constexpr bool type_has_shared_from_this(const std::enable_shared_from_this *) { + return true; +} + +struct guarded_delete { + std::weak_ptr released_ptr; // Trick to keep the smart_holder memory footprint small. + std::function del_fun; // Rare case. + void (*del_ptr)(void *); // Common case. + bool use_del_fun; + bool armed_flag; + guarded_delete(std::function &&del_fun, bool armed_flag) + : del_fun{std::move(del_fun)}, del_ptr{nullptr}, use_del_fun{true}, + armed_flag{armed_flag} {} + guarded_delete(void (*del_ptr)(void *), bool armed_flag) + : del_ptr{del_ptr}, use_del_fun{false}, armed_flag{armed_flag} {} + void operator()(void *raw_ptr) const { + if (armed_flag) { + if (use_del_fun) { + del_fun(raw_ptr); + } else { + del_ptr(raw_ptr); + } + } + } +}; + +template ::value, int>::type = 0> +inline void builtin_delete_if_destructible(void *raw_ptr) { + std::default_delete{}(static_cast(raw_ptr)); +} + +template ::value, int>::type = 0> +inline void builtin_delete_if_destructible(void *) { + // This noop operator is needed to avoid a compilation error (for `delete raw_ptr;`), but + // throwing an exception from a destructor will std::terminate the process. Therefore the + // runtime check for lifetime-management correctness is implemented elsewhere (in + // ensure_pointee_is_destructible()). +} + +template +guarded_delete make_guarded_builtin_delete(bool armed_flag) { + return guarded_delete(builtin_delete_if_destructible, armed_flag); +} + +template +struct custom_deleter { + D deleter; + explicit custom_deleter(D &&deleter) : deleter{std::forward(deleter)} {} + void operator()(void *raw_ptr) { deleter(static_cast(raw_ptr)); } +}; + +template +guarded_delete make_guarded_custom_deleter(D &&uqp_del, bool armed_flag) { + return guarded_delete( + std::function(custom_deleter(std::forward(uqp_del))), armed_flag); +} + +template +inline bool is_std_default_delete(const std::type_info &rtti_deleter) { + return rtti_deleter == typeid(std::default_delete) + || rtti_deleter == typeid(std::default_delete); +} + +struct smart_holder { + const std::type_info *rtti_uqp_del = nullptr; + std::shared_ptr vptr; + bool vptr_is_using_noop_deleter : 1; + bool vptr_is_using_builtin_delete : 1; + bool vptr_is_external_shared_ptr : 1; + bool is_populated : 1; + bool is_disowned : 1; + + // Design choice: smart_holder is movable but not copyable. + smart_holder(smart_holder &&) = default; + smart_holder(const smart_holder &) = delete; + smart_holder &operator=(smart_holder &&) = delete; + smart_holder &operator=(const smart_holder &) = delete; + + smart_holder() + : vptr_is_using_noop_deleter{false}, vptr_is_using_builtin_delete{false}, + vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false} {} + + bool has_pointee() const { return vptr != nullptr; } + + template + static void ensure_pointee_is_destructible(const char *context) { + if (!std::is_destructible::value) { + throw std::invalid_argument(std::string("Pointee is not destructible (") + context + + ")."); + } + } + + void ensure_is_populated(const char *context) const { + if (!is_populated) { + throw std::runtime_error(std::string("Unpopulated holder (") + context + ")."); + } + } + void ensure_is_not_disowned(const char *context) const { + if (is_disowned) { + throw std::runtime_error(std::string("Holder was disowned already (") + context + + ")."); + } + } + + void ensure_vptr_is_using_builtin_delete(const char *context) const { + if (vptr_is_external_shared_ptr) { + throw std::invalid_argument(std::string("Cannot disown external shared_ptr (") + + context + ")."); + } + if (vptr_is_using_noop_deleter) { + throw std::invalid_argument(std::string("Cannot disown non-owning holder (") + context + + ")."); + } + if (!vptr_is_using_builtin_delete) { + throw std::invalid_argument(std::string("Cannot disown custom deleter (") + context + + ")."); + } + } + + template + void ensure_compatible_rtti_uqp_del(const char *context) const { + const std::type_info *rtti_requested = &typeid(D); + if (!rtti_uqp_del) { + if (!is_std_default_delete(*rtti_requested)) { + throw std::invalid_argument(std::string("Missing unique_ptr deleter (") + context + + ")."); + } + ensure_vptr_is_using_builtin_delete(context); + } else if (!(*rtti_requested == *rtti_uqp_del) + && !(vptr_is_using_builtin_delete + && is_std_default_delete(*rtti_requested))) { + throw std::invalid_argument(std::string("Incompatible unique_ptr deleter (") + context + + ")."); + } + } + + void ensure_has_pointee(const char *context) const { + if (!has_pointee()) { + throw std::invalid_argument(std::string("Disowned holder (") + context + ")."); + } + } + + void ensure_use_count_1(const char *context) const { + if (vptr == nullptr) { + throw std::invalid_argument(std::string("Cannot disown nullptr (") + context + ")."); + } + // In multithreaded environments accessing use_count can lead to + // race conditions, but in the context of Python it is a bug (elsewhere) + // if the Global Interpreter Lock (GIL) is not being held when this code + // is reached. + // PYBIND11:REMINDER: This may need to be protected by a mutex in free-threaded Python. + if (vptr.use_count() != 1) { + throw std::invalid_argument(std::string("Cannot disown use_count != 1 (") + context + + ")."); + } + } + + void reset_vptr_deleter_armed_flag(bool armed_flag) const { + auto *vptr_del_ptr = std::get_deleter(vptr); + if (vptr_del_ptr == nullptr) { + throw std::runtime_error( + "smart_holder::reset_vptr_deleter_armed_flag() called in an invalid context."); + } + vptr_del_ptr->armed_flag = armed_flag; + } + + // Caller is responsible for precondition: ensure_compatible_rtti_uqp_del() must succeed. + template + std::unique_ptr extract_deleter(const char *context) const { + const auto *gd = std::get_deleter(vptr); + if (gd && gd->use_del_fun) { + const auto &custom_deleter_ptr = gd->del_fun.template target>(); + if (custom_deleter_ptr == nullptr) { + throw std::runtime_error( + std::string("smart_holder::extract_deleter() precondition failure (") + context + + ")."); + } + static_assert(std::is_copy_constructible::value, + "Required for compatibility with smart_holder functionality."); + return std::unique_ptr(new D(custom_deleter_ptr->deleter)); + } + return nullptr; + } + + static smart_holder from_raw_ptr_unowned(void *raw_ptr) { + smart_holder hld; + hld.vptr.reset(raw_ptr, [](void *) {}); + hld.vptr_is_using_noop_deleter = true; + hld.is_populated = true; + return hld; + } + + template + T *as_raw_ptr_unowned() const { + return static_cast(vptr.get()); + } + + template + static smart_holder from_raw_ptr_take_ownership(T *raw_ptr, bool void_cast_raw_ptr = false) { + ensure_pointee_is_destructible("from_raw_ptr_take_ownership"); + smart_holder hld; + auto gd = make_guarded_builtin_delete(true); + if (void_cast_raw_ptr) { + hld.vptr.reset(static_cast(raw_ptr), std::move(gd)); + } else { + hld.vptr.reset(raw_ptr, std::move(gd)); + } + hld.vptr_is_using_builtin_delete = true; + hld.is_populated = true; + return hld; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void disown() { + reset_vptr_deleter_armed_flag(false); + is_disowned = true; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void reclaim_disowned() { + reset_vptr_deleter_armed_flag(true); + is_disowned = false; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_disowned() { vptr.reset(); } + + void ensure_can_release_ownership(const char *context = "ensure_can_release_ownership") const { + ensure_is_not_disowned(context); + ensure_vptr_is_using_builtin_delete(context); + ensure_use_count_1(context); + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_ownership() { + reset_vptr_deleter_armed_flag(false); + release_disowned(); + } + + template + static smart_holder from_unique_ptr(std::unique_ptr &&unq_ptr, + void *void_ptr = nullptr) { + smart_holder hld; + hld.rtti_uqp_del = &typeid(D); + hld.vptr_is_using_builtin_delete = is_std_default_delete(*hld.rtti_uqp_del); + guarded_delete gd{nullptr, false}; + if (hld.vptr_is_using_builtin_delete) { + gd = make_guarded_builtin_delete(true); + } else { + gd = make_guarded_custom_deleter(std::move(unq_ptr.get_deleter()), true); + } + if (void_ptr != nullptr) { + hld.vptr.reset(void_ptr, std::move(gd)); + } else { + hld.vptr.reset(unq_ptr.get(), std::move(gd)); + } + (void) unq_ptr.release(); + hld.is_populated = true; + return hld; + } + + template + static smart_holder from_shared_ptr(std::shared_ptr shd_ptr) { + smart_holder hld; + hld.vptr = std::static_pointer_cast(shd_ptr); + hld.vptr_is_external_shared_ptr = true; + hld.is_populated = true; + return hld; + } + + template + std::shared_ptr as_shared_ptr() const { + return std::static_pointer_cast(vptr); + } +}; + +} // namespace memory +} // namespace pybindit diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8751364d4c..9618b2181d 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -9,13 +9,17 @@ #pragma once +#include #include +#include #include "common.h" #include "cpp_conduit.h" #include "descr.h" +#include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" #include "typeid.h" +#include "using_smart_holder.h" #include "value_and_holder.h" #include @@ -507,6 +511,361 @@ inline PyThreadState *get_thread_state_unchecked() { void keep_alive_impl(handle nurse, handle patient); inline PyObject *make_new_instance(PyTypeObject *type); +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); + +PYBIND11_NAMESPACE_BEGIN(smart_holder_type_caster_support) + +struct value_and_holder_helper { + value_and_holder loaded_v_h; + + bool have_holder() const { + return loaded_v_h.vh != nullptr && loaded_v_h.holder_constructed(); + } + + smart_holder &holder() const { return loaded_v_h.holder(); } + + void throw_if_uninitialized_or_disowned_holder(const char *typeid_name) const { + static const std::string missing_value_msg = "Missing value for wrapped C++ type `"; + if (!holder().is_populated) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance is uninitialized."); + } + if (!holder().has_pointee()) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance was disowned."); + } + } + + void throw_if_uninitialized_or_disowned_holder(const std::type_info &type_info) const { + throw_if_uninitialized_or_disowned_holder(type_info.name()); + } + + // have_holder() must be true or this function will fail. + void throw_if_instance_is_currently_owned_by_shared_ptr() const { + auto *vptr_gd_ptr = std::get_deleter(holder().vptr); + if (vptr_gd_ptr != nullptr && !vptr_gd_ptr->released_ptr.expired()) { + throw value_error("Python instance is currently owned by a std::shared_ptr."); + } + } + + void *get_void_ptr_or_nullptr() const { + if (have_holder()) { + auto &hld = holder(); + if (hld.is_populated && hld.has_pointee()) { + return hld.template as_raw_ptr_unowned(); + } + } + return nullptr; + } +}; + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const std::pair &st) { + if (policy == return_value_policy::copy) { + throw cast_error("return_value_policy::copy is invalid for unique_ptr."); + } + if (!src) { + return none().release(); + } + void *src_raw_void_ptr = const_cast(st.first); + assert(st.second != nullptr); + const detail::type_info *tinfo = st.second; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + auto *self_life_support + = dynamic_raw_ptr_cast_if_possible(src.get()); + if (self_life_support != nullptr) { + value_and_holder &v_h = self_life_support->v_h; + if (v_h.inst != nullptr && v_h.vh != nullptr) { + auto &holder = v_h.holder(); + if (!holder.is_disowned) { + pybind11_fail("smart_holder_from_unique_ptr: unexpected " + "smart_holder.is_disowned failure."); + } + // Critical transfer-of-ownership section. This must stay together. + self_life_support->deactivate_life_support(); + holder.reclaim_disowned(); + (void) src.release(); + // Critical section end. + return existing_inst; + } + } + throw cast_error("Invalid unique_ptr: another instance owns this pointer already."); + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + if (static_cast(src.get()) == src_raw_void_ptr) { + // This is a multiple-inheritance situation that is incompatible with the current + // shared_from_this handling (see PR #3023). Is there a better solution? + src_raw_void_ptr = nullptr; + } + auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const std::pair &st) { + return smart_holder_from_unique_ptr( + std::unique_ptr(const_cast(src.release()), + std::move(src.get_deleter())), // Const2Mutbl + policy, + parent, + st); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const std::pair &st) { + switch (policy) { + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + break; + case return_value_policy::take_ownership: + throw cast_error("Invalid return_value_policy for shared_ptr (take_ownership)."); + case return_value_policy::copy: + case return_value_policy::move: + break; + case return_value_policy::reference: + throw cast_error("Invalid return_value_policy for shared_ptr (reference)."); + case return_value_policy::reference_internal: + break; + } + if (!src) { + return none().release(); + } + + auto src_raw_ptr = src.get(); + assert(st.second != nullptr); + void *src_raw_void_ptr = static_cast(src_raw_ptr); + const detail::type_info *tinfo = st.second; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder. + // PYBIND11:REMINDER: MISSING: keep_alive. + return existing_inst; + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + auto smhldr + = smart_holder::from_shared_ptr(std::shared_ptr(src, const_cast(st.first))); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const std::pair &st) { + return smart_holder_from_shared_ptr(std::const_pointer_cast(src), // Const2Mutbl + policy, + parent, + st); +} + +struct shared_ptr_parent_life_support { + PyObject *parent; + explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} { + Py_INCREF(parent); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(parent); + } +}; + +struct shared_ptr_trampoline_self_life_support { + PyObject *self; + explicit shared_ptr_trampoline_self_life_support(instance *inst) + : self{reinterpret_cast(inst)} { + gil_scoped_acquire gil; + Py_INCREF(self); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(self); + } +}; + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + return std::unique_ptr(raw_ptr); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + pybind11_fail("smart_holder_type_casters: deleter is not default constructible and no" + " instance available to return."); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template +struct load_helper : value_and_holder_helper { + bool was_populated = false; + bool python_instance_is_alias = false; + + void maybe_set_python_instance_is_alias(handle src) { + if (was_populated) { + python_instance_is_alias = reinterpret_cast(src.ptr())->is_alias; + } + } + + static std::shared_ptr make_shared_ptr_with_responsible_parent(T *raw_ptr, handle parent) { + return std::shared_ptr(raw_ptr, shared_ptr_parent_life_support(parent.ptr())); + } + + std::shared_ptr load_as_shared_ptr(void *void_raw_ptr, + handle responsible_parent = nullptr) const { + if (!have_holder()) { + return nullptr; + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + smart_holder &hld = holder(); + hld.ensure_is_not_disowned("load_as_shared_ptr"); + if (hld.vptr_is_using_noop_deleter) { + if (responsible_parent) { + return make_shared_ptr_with_responsible_parent(static_cast(void_raw_ptr), + responsible_parent); + } + throw std::runtime_error("Non-owning holder (load_as_shared_ptr)."); + } + auto *type_raw_ptr = static_cast(void_raw_ptr); + if (python_instance_is_alias) { + auto *vptr_gd_ptr = std::get_deleter(hld.vptr); + if (vptr_gd_ptr != nullptr) { + std::shared_ptr released_ptr = vptr_gd_ptr->released_ptr.lock(); + if (released_ptr) { + return std::shared_ptr(released_ptr, type_raw_ptr); + } + std::shared_ptr to_be_released( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + vptr_gd_ptr->released_ptr = to_be_released; + return to_be_released; + } + auto *sptsls_ptr = std::get_deleter(hld.vptr); + if (sptsls_ptr != nullptr) { + // This code is reachable only if there are multiple registered_instances for the + // same pointee. + if (reinterpret_cast(loaded_v_h.inst) == sptsls_ptr->self) { + pybind11_fail("smart_holder_type_caster_support load_as_shared_ptr failure: " + "loaded_v_h.inst == sptsls_ptr->self"); + } + } + if (sptsls_ptr != nullptr + || !pybindit::memory::type_has_shared_from_this(type_raw_ptr)) { + return std::shared_ptr( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + } + if (hld.vptr_is_external_shared_ptr) { + pybind11_fail("smart_holder_type_casters load_as_shared_ptr failure: not " + "implemented: trampoline-self-life-support for external shared_ptr " + "to type inheriting from std::enable_shared_from_this."); + } + pybind11_fail( + "smart_holder_type_casters: load_as_shared_ptr failure: internal inconsistency."); + } + std::shared_ptr void_shd_ptr = hld.template as_shared_ptr(); + return std::shared_ptr(void_shd_ptr, type_raw_ptr); + } + + template + std::unique_ptr load_as_unique_ptr(void *raw_void_ptr, + const char *context = "load_as_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + throw_if_instance_is_currently_owned_by_shared_ptr(); + holder().ensure_is_not_disowned(context); + holder().template ensure_compatible_rtti_uqp_del(context); + holder().ensure_use_count_1(context); + + T *raw_type_ptr = static_cast(raw_void_ptr); + + auto *self_life_support + = dynamic_raw_ptr_cast_if_possible(raw_type_ptr); + if (self_life_support == nullptr && python_instance_is_alias) { + throw value_error("Alias class (also known as trampoline) does not inherit from " + "py::trampoline_self_life_support, therefore the ownership of this " + "instance cannot safely be transferred to C++."); + } + + std::unique_ptr extracted_deleter = holder().template extract_deleter(context); + + // Critical transfer-of-ownership section. This must stay together. + if (self_life_support != nullptr) { + holder().disown(); + } else { + holder().release_ownership(); + } + auto result = unique_with_deleter(raw_type_ptr, std::move(extracted_deleter)); + if (self_life_support != nullptr) { + self_life_support->activate_life_support(loaded_v_h); + } else { + void *value_void_ptr = loaded_v_h.value_ptr(); + loaded_v_h.value_ptr() = nullptr; + deregister_instance(loaded_v_h.inst, value_void_ptr, loaded_v_h.type); + } + // Critical section end. + + return result; + } + + // This assumes load_as_shared_ptr succeeded(), and the returned shared_ptr is still alive. + // The returned unique_ptr is meant to never expire (the behavior is undefined otherwise). + template + std::unique_ptr + load_as_const_unique_ptr(T *raw_type_ptr, const char *context = "load_as_const_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + holder().template ensure_compatible_rtti_uqp_del(context); + return unique_with_deleter( + raw_type_ptr, std::move(holder().template extract_deleter(context))); + } +}; + +PYBIND11_NAMESPACE_END(smart_holder_type_caster_support) + class type_caster_generic { public: PYBIND11_NOINLINE explicit type_caster_generic(const std::type_info &type_info) @@ -611,6 +970,15 @@ class type_caster_generic { // Base methods for generic caster; there are overridden in copyable_holder_caster void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + smart_holder_type_caster_support::value_and_holder_helper v_h_helper; + v_h_helper.loaded_v_h = v_h; + if (v_h_helper.have_holder()) { + v_h_helper.throw_if_uninitialized_or_disowned_holder(cpptype->name()); + value = v_h_helper.holder().template as_raw_ptr_unowned(); + return; + } + } auto *&vptr = v_h.value_ptr(); // Lazy allocation for unallocated values: if (vptr == nullptr) { diff --git a/include/pybind11/detail/using_smart_holder.h b/include/pybind11/detail/using_smart_holder.h new file mode 100644 index 0000000000..57f99b95f3 --- /dev/null +++ b/include/pybind11/detail/using_smart_holder.h @@ -0,0 +1,22 @@ +// Copyright (c) 2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" +#include "struct_smart_holder.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +using pybindit::memory::smart_holder; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +using is_smart_holder = std::is_same; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index ca37d70ad2..64c55cc595 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -7,6 +7,7 @@ #include "common.h" #include +#include #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 8a1739e193..5564f040e5 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -10,8 +10,10 @@ #pragma once #include "detail/class.h" +#include "detail/dynamic_raw_ptr_cast_if_possible.h" #include "detail/exception_translation.h" #include "detail/init.h" +#include "detail/using_smart_holder.h" #include "attr.h" #include "gil.h" #include "gil_safe_call_once.h" @@ -1444,8 +1446,8 @@ class generic_type : public object { tinfo->dealloc = rec.dealloc; tinfo->simple_type = true; tinfo->simple_ancestors = true; - tinfo->default_holder = rec.default_holder; tinfo->module_local = rec.module_local; + tinfo->holder_enum_v = rec.holder_enum_v; with_internals([&](internals &internals) { auto tindex = std::type_index(*rec.type); @@ -1618,6 +1620,240 @@ auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)( return pmf; } +PYBIND11_NAMESPACE_BEGIN(detail) + +// Helper for the property_cpp_function static member functions below. +// The only purpose of these functions is to support .def_readonly & .def_readwrite. +// In this context, the PM template parameter is certain to be a Pointer to a Member. +// The main purpose of must_be_member_function_pointer is to make this obvious, and to guard +// against accidents. As a side-effect, it also explains why the syntactical overhead for +// perfect forwarding is not needed. +template +using must_be_member_function_pointer = enable_if_t::value, int>; + +// Note that property_cpp_function is intentionally in the main pybind11 namespace, +// because user-defined specializations could be useful. + +// Classic (non-smart_holder) implementations for .def_readonly and .def_readwrite +// getter and setter functions. +// WARNING: This classic implementation can lead to dangling pointers for raw pointer members. +// See test_ptr() in tests/test_class_sh_property.py +// However, this implementation works as-is (and safely) for smart_holder std::shared_ptr members. +template +struct property_cpp_function_classic { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + return cpp_function([pm](const T &c) -> const D & { return c.*pm; }, is_method(hdl)); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function : detail::property_cpp_function_classic {}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct both_t_and_d_use_type_caster_base : std::false_type {}; + +// `T` is assumed to be equivalent to `intrinsic_t`. +// `D` is may or may not be equivalent to `intrinsic_t`. +template +struct both_t_and_d_use_type_caster_base< + T, + D, + enable_if_t, type_caster>, + std::is_base_of>, make_caster>>::value>> + : std::true_type {}; + +// Specialization for raw pointer members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// WARNING: Like the classic implementation, this implementation can lead to dangling pointers. +// See test_ptr() in tests/test_class_sh_property.py +// However, the read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the raw pointer member. +template +struct property_cpp_function_sh_raw_ptr_member { + using drp = typename std::remove_pointer::type; + + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + D ptr = (*c_sp).*pm; + return std::shared_ptr(c_sp, ptr); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, D value) { c.*pm = std::forward(std::move(value)); }, + is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for members held by-value, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// The read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the member. +template +struct property_cpp_function_sh_member_held_by_value { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr::type> { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr::type>(c_sp, + &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr(c_sp, &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for std::unique_ptr members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// read disowns the member unique_ptr. +// write disowns the passed Python object. +// readonly is disabled (static_assert) because there is no safe & intuitive way to make the member +// accessible as a Python object without disowning the member unique_ptr. A .def_readonly disowning +// the unique_ptr member is deemed highly prone to misunderstandings. +template +struct property_cpp_function_sh_unique_ptr_member { + template = 0> + static cpp_function readonly(PM, const handle &) { + static_assert(!is_instantiation::value, + "def_readonly cannot be used for std::unique_ptr members."); + return cpp_function{}; // Unreachable. + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> D { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return D{std::move(c_sp.get()->*pm)}; + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, D &&value) { c.*pm = std::move(value); }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_raw_ptr_member {}; + +template +struct property_cpp_function, + std::is_array, + detail::is_instantiation, + detail::is_instantiation>, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_member_held_by_value {}; + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_unique_ptr_member {}; + +#if defined(PYBIND11_USE_SMART_HOLDER_AS_DEFAULT) +// NOTE: THIS IS MEANT FOR STRESS-TESTING ONLY! +// As of PR #5257, for production use, there is no longer a strong reason to make +// smart_holder the default holder: +// Simply use `py::classh` (see below) instead of `py::class_` as needed. +// Running the pybind11 unit tests with smart_holder as the default holder is to ensure +// that `py::smart_holder` / `py::classh` is backward-compatible with all pre-existing +// functionality. +template +using default_holder_type = smart_holder; +#else +template +using default_holder_type = std::unique_ptr; +#endif + template class class_ : public detail::generic_type { template @@ -1634,7 +1870,7 @@ class class_ : public detail::generic_type { using type = type_; using type_alias = detail::exactly_one_t; constexpr static bool has_alias = !std::is_void::value; - using holder_type = detail::exactly_one_t, options...>; + using holder_type = detail::exactly_one_t, options...>; static_assert(detail::all_of...>::value, "Unknown/invalid class_ template parameters provided"); @@ -1665,7 +1901,16 @@ class class_ : public detail::generic_type { record.type_align = alignof(conditional_t &); record.holder_size = sizeof(holder_type); record.init_instance = init_instance; - record.default_holder = detail::is_instantiation::value; + + if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_unique_ptr; + } else if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_shared_ptr; + } else if (std::is_same::value) { + record.holder_enum_v = detail::holder_enum_t::smart_holder; + } else { + record.holder_enum_v = detail::holder_enum_t::custom_holder; + } set_operator_new(&record); @@ -1804,9 +2049,11 @@ class class_ : public detail::generic_type { class_ &def_readwrite(const char *name, D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readwrite() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)), - fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); - def_property(name, fget, fset, return_value_policy::reference_internal, extra...); + def_property(name, + property_cpp_function::read(pm, *this), + property_cpp_function::write(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1814,8 +2061,10 @@ class class_ : public detail::generic_type { class_ &def_readonly(const char *name, const D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readonly() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)); - def_property_readonly(name, fget, return_value_policy::reference_internal, extra...); + def_property_readonly(name, + property_cpp_function::readonly(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1992,6 +2241,8 @@ class class_ : public detail::generic_type { /// instance. Should be called as soon as the `type` value_ptr is set for an instance. Takes /// an optional pointer to an existing holder to use; if not specified and the instance is /// `.owned`, a new holder will be constructed to manage the value pointer. + template ::value, int> = 0> static void init_instance(detail::instance *inst, const void *holder_ptr) { auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); if (!v_h.instance_registered()) { @@ -2001,6 +2252,65 @@ class class_ : public detail::generic_type { init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr()); } + template + static bool try_initialization_using_shared_from_this(holder_type *, WrappedType *, ...) { + return false; + } + + // Adopting existing approach used by type_caster_base, although it leads to somewhat fuzzy + // ownership semantics: if we detected via shared_from_this that a shared_ptr exists already, + // it is reused, irrespective of the return_value_policy in effect. + // "SomeBaseOfWrappedType" is needed because std::enable_shared_from_this is not necessarily a + // direct base of WrappedType. + template + static bool try_initialization_using_shared_from_this( + holder_type *uninitialized_location, + WrappedType *value_ptr_w_t, + const std::enable_shared_from_this *) { + auto shd_ptr = std::dynamic_pointer_cast( + detail::try_get_shared_from_this(value_ptr_w_t)); + if (!shd_ptr) { + return false; + } + // Note: inst->owned ignored. + new (uninitialized_location) holder_type(holder_type::from_shared_ptr(shd_ptr)); + return true; + } + + template ::value, int> = 0> + static void init_instance(detail::instance *inst, const void *holder_const_void_ptr) { + // Need for const_cast is a consequence of the type_info::init_instance type: + // void (*init_instance)(instance *, const void *); + auto *holder_void_ptr = const_cast(holder_const_void_ptr); + + auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); + if (!v_h.instance_registered()) { + register_instance(inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(); + } + auto *uninitialized_location = std::addressof(v_h.holder()); + auto *value_ptr_w_t = v_h.value_ptr(); + // Try downcast from `type` to `type_alias`: + inst->is_alias + = detail::dynamic_raw_ptr_cast_if_possible(value_ptr_w_t) != nullptr; + if (holder_void_ptr) { + // Note: inst->owned ignored. + auto *holder_ptr = static_cast(holder_void_ptr); + new (uninitialized_location) holder_type(std::move(*holder_ptr)); + } else if (!try_initialization_using_shared_from_this( + uninitialized_location, value_ptr_w_t, value_ptr_w_t)) { + if (inst->owned) { + new (uninitialized_location) holder_type(holder_type::from_raw_ptr_take_ownership( + value_ptr_w_t, /*void_cast_raw_ptr*/ inst->is_alias)); + } else { + new (uninitialized_location) + holder_type(holder_type::from_raw_ptr_unowned(value_ptr_w_t)); + } + } + v_h.set_holder_constructed(); + } + // Deallocates an instance; via holder, if constructed; otherwise via operator delete. // NOTE: The Python error indicator needs to cleared BEFORE this function is called. // This is because we could be deallocating while cleaning up after a Python exception. @@ -2066,6 +2376,14 @@ class class_ : public detail::generic_type { } }; +// Supports easier switching between py::class_ and py::class_: +// users can simply replace the `_` in `class_` with `h` or vice versa. +template +class classh : public class_ { +public: + using class_::class_; +}; + /// Binds an existing constructor taking arguments Args... template detail::initimpl::constructor init() { diff --git a/include/pybind11/smart_holder.h b/include/pybind11/smart_holder.h new file mode 100644 index 0000000000..5f568a5529 --- /dev/null +++ b/include/pybind11/smart_holder.h @@ -0,0 +1,14 @@ +// Copyright (c) 2021-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "pybind11.h" + +// Legacy macros introduced with smart_holder_type_casters implementation in 2021. +// Deprecated. +#define PYBIND11_TYPE_CASTER_BASE_HOLDER(...) +#define PYBIND11_SMART_HOLDER_TYPE_CASTERS(...) +#define PYBIND11_SH_AVL(...) // "Smart_Holder if AVaiLable" +#define PYBIND11_SH_DEF(...) // "Smart_Holder if DEFault" diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index af3a47f39c..3eb1e53f45 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -487,7 +487,7 @@ PYBIND11_NAMESPACE_END(detail) // // std::vector // -template , typename... Args> +template , typename... Args> class_ bind_vector(handle scope, std::string const &name, Args &&...args) { using Class_ = class_; @@ -730,7 +730,7 @@ str format_message_key_error(const KeyType &key) { PYBIND11_NAMESPACE_END(detail) -template , typename... Args> +template , typename... Args> class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; diff --git a/include/pybind11/trampoline_self_life_support.h b/include/pybind11/trampoline_self_life_support.h new file mode 100644 index 0000000000..484045bb17 --- /dev/null +++ b/include/pybind11/trampoline_self_life_support.h @@ -0,0 +1,60 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "detail/common.h" +#include "detail/using_smart_holder.h" +#include "detail/value_and_holder.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); +PYBIND11_NAMESPACE_END(detail) + +// The original core idea for this struct goes back to PyCLIF: +// https://github.com/google/clif/blob/07f95d7e69dca2fcf7022978a55ef3acff506c19/clif/python/runtime.cc#L37 +// URL provided here mainly to give proper credit. +struct trampoline_self_life_support { + detail::value_and_holder v_h; + + trampoline_self_life_support() = default; + + void activate_life_support(const detail::value_and_holder &v_h_) { + Py_INCREF((PyObject *) v_h_.inst); + v_h = v_h_; + } + + void deactivate_life_support() { + Py_DECREF((PyObject *) v_h.inst); + v_h = detail::value_and_holder(); + } + + ~trampoline_self_life_support() { + if (v_h.inst != nullptr && v_h.vh != nullptr) { + void *value_void_ptr = v_h.value_ptr(); + if (value_void_ptr != nullptr) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + v_h.value_ptr() = nullptr; + v_h.holder().release_disowned(); + detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type); + Py_DECREF((PyObject *) v_h.inst); // Must be after deregister. + PyGILState_Release(threadstate); + } + } + } + + // For the next two, the default implementations generate undefined behavior (ASAN failures + // manually verified). The reason is that v_h needs to be kept default-initialized. + trampoline_self_life_support(const trampoline_self_life_support &) {} + trampoline_self_life_support(trampoline_self_life_support &&) noexcept {} + + // These should never be needed (please provide test cases if you think they are). + trampoline_self_life_support &operator=(const trampoline_self_life_support &) = delete; + trampoline_self_life_support &operator=(trampoline_self_life_support &&) = delete; +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index da396f0984..e2fab9c13d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,6 +116,23 @@ set(PYBIND11_TEST_FILES test_chrono test_class test_class_release_gil_before_calling_cpp_dtor + test_class_sh_basic + test_class_sh_disowning + test_class_sh_disowning_mi + test_class_sh_factory_constructors + test_class_sh_inheritance + test_class_sh_mi_thunks + test_class_sh_property + test_class_sh_property_non_owning + test_class_sh_shared_ptr_copy_move + test_class_sh_trampoline_basic + test_class_sh_trampoline_self_life_support + test_class_sh_trampoline_shared_from_this + test_class_sh_trampoline_shared_ptr_cpp_arg + test_class_sh_trampoline_unique_ptr + test_class_sh_unique_ptr_custom_deleter + test_class_sh_unique_ptr_member + test_class_sh_virtual_py_cpp_mix test_const_name test_constants_and_functions test_copy_move @@ -589,6 +606,9 @@ add_custom_command( ${CMAKE_CURRENT_BINARY_DIR}/sosize-$.txt) if(NOT PYBIND11_CUDA_TESTS) + # Test pure C++ code (not depending on Python). Provides the `test_pure_cpp` target. + add_subdirectory(pure_cpp) + # Test embedding the interpreter. Provides the `cpptest` target. add_subdirectory(test_embed) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 977db75c63..f50fb15304 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -43,8 +43,10 @@ "include/pybind11/options.h", "include/pybind11/pybind11.h", "include/pybind11/pytypes.h", + "include/pybind11/smart_holder.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/trampoline_self_life_support.h", "include/pybind11/type_caster_pyobject_ptr.h", "include/pybind11/typing.h", "include/pybind11/warnings.h", @@ -62,10 +64,13 @@ "include/pybind11/detail/common.h", "include/pybind11/detail/cpp_conduit.h", "include/pybind11/detail/descr.h", + "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", + "include/pybind11/detail/struct_smart_holder.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", + "include/pybind11/detail/using_smart_holder.h", "include/pybind11/detail/value_and_holder.h", "include/pybind11/detail/exception_translation.h", } @@ -127,6 +132,7 @@ "LICENSE", "MANIFEST.in", "README.rst", + "README_smart_holder.rst", "PKG-INFO", "SECURITY.md", } diff --git a/tests/pure_cpp/CMakeLists.txt b/tests/pure_cpp/CMakeLists.txt new file mode 100644 index 0000000000..17be74b7f0 --- /dev/null +++ b/tests/pure_cpp/CMakeLists.txt @@ -0,0 +1,20 @@ +find_package(Catch 2.13.2) + +if(CATCH_FOUND) + message(STATUS "Building pure C++ tests (not depending on Python) using Catch v${CATCH_VERSION}") +else() + message(STATUS "Catch not detected. Interpreter tests will be skipped. Install Catch headers" + " manually or use `cmake -DDOWNLOAD_CATCH=ON` to fetch them automatically.") + return() +endif() + +add_executable(smart_holder_poc_test smart_holder_poc_test.cpp) +pybind11_enable_warnings(smart_holder_poc_test) +target_link_libraries(smart_holder_poc_test PRIVATE pybind11::headers Catch2::Catch2) + +add_custom_target( + test_pure_cpp + COMMAND "$" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + +add_dependencies(check test_pure_cpp) diff --git a/tests/pure_cpp/smart_holder_poc.h b/tests/pure_cpp/smart_holder_poc.h new file mode 100644 index 0000000000..320311b7d6 --- /dev/null +++ b/tests/pure_cpp/smart_holder_poc.h @@ -0,0 +1,51 @@ +// Copyright (c) 2020-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "pybind11/detail/struct_smart_holder.h" + +namespace pybindit { +namespace memory { +namespace smart_holder_poc { // Proof-of-Concept implementations. + +template +T &as_lvalue_ref(const smart_holder &hld) { + static const char *context = "as_lvalue_ref"; + hld.ensure_is_populated(context); + hld.ensure_has_pointee(context); + return *hld.as_raw_ptr_unowned(); +} + +template +T &&as_rvalue_ref(const smart_holder &hld) { + static const char *context = "as_rvalue_ref"; + hld.ensure_is_populated(context); + hld.ensure_has_pointee(context); + return std::move(*hld.as_raw_ptr_unowned()); +} + +template +T *as_raw_ptr_release_ownership(smart_holder &hld, + const char *context = "as_raw_ptr_release_ownership") { + hld.ensure_can_release_ownership(context); + T *raw_ptr = hld.as_raw_ptr_unowned(); + hld.release_ownership(); + return raw_ptr; +} + +template > +std::unique_ptr as_unique_ptr(smart_holder &hld) { + static const char *context = "as_unique_ptr"; + hld.ensure_compatible_rtti_uqp_del(context); + hld.ensure_use_count_1(context); + T *raw_ptr = hld.as_raw_ptr_unowned(); + hld.release_ownership(); + // KNOWN DEFECT (see PR #4850): Does not copy the deleter. + return std::unique_ptr(raw_ptr); +} + +} // namespace smart_holder_poc +} // namespace memory +} // namespace pybindit diff --git a/tests/pure_cpp/smart_holder_poc_test.cpp b/tests/pure_cpp/smart_holder_poc_test.cpp new file mode 100644 index 0000000000..24ab643eeb --- /dev/null +++ b/tests/pure_cpp/smart_holder_poc_test.cpp @@ -0,0 +1,415 @@ +#include "smart_holder_poc.h" + +#include +#include +#include +#include + +// Catch uses _ internally, which breaks gettext style defines +#ifdef _ +# undef _ +#endif + +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +using pybindit::memory::smart_holder; +namespace poc = pybindit::memory::smart_holder_poc; + +namespace helpers { + +struct movable_int { + int valu; + explicit movable_int(int v) : valu{v} {} + movable_int(movable_int &&other) noexcept : valu(other.valu) { other.valu = 91; } +}; + +template +struct functor_builtin_delete { + void operator()(T *ptr) { delete ptr; } +#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8) \ + || (defined(__clang_major__) && __clang_major__ == 3 && __clang_minor__ == 6) + // Workaround for these errors: + // gcc 4.8.5: too many initializers for 'helpers::functor_builtin_delete' + // clang 3.6: excess elements in struct initializer + functor_builtin_delete() = default; + functor_builtin_delete(const functor_builtin_delete &) {} + functor_builtin_delete(functor_builtin_delete &&) {} +#endif +}; + +template +struct functor_other_delete : functor_builtin_delete {}; + +struct indestructible_int { + int valu; + explicit indestructible_int(int v) : valu{v} {} + +private: + ~indestructible_int() = default; +}; + +struct base { + virtual int get() { return 10; } + virtual ~base() = default; +}; + +struct derived : public base { + int get() override { return 100; } +}; + +} // namespace helpers + +TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(*hld.as_raw_ptr_unowned() == 19); +} + +TEST_CASE("from_raw_ptr_unowned+as_lvalue_ref", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_raw_ptr_unowned+as_rvalue_ref", "[S]") { + helpers::movable_int orig(19); + { + auto hld = smart_holder::from_raw_ptr_unowned(&orig); + helpers::movable_int othr(poc::as_rvalue_ref(hld)); + REQUIRE(othr.valu == 19); + REQUIRE(orig.valu == 91); + } +} + +TEST_CASE("from_raw_ptr_unowned+as_raw_ptr_release_ownership", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown non-owning holder (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_unique_ptr", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown non-owning holder (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_unique_ptr_with_deleter", "[E]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_unowned+as_shared_ptr", "[S]") { + static int value = 19; + auto hld = smart_holder::from_raw_ptr_unowned(&value); + REQUIRE(*hld.as_shared_ptr() == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_lvalue_ref", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + REQUIRE(hld.has_pointee()); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership1", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto new_owner = std::unique_ptr(poc::as_raw_ptr_release_ownership(hld)); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_raw_ptr_release_ownership2", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown use_count != 1 (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr1", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr2", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown use_count != 1 (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_unique_ptr_with_deleter", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+reclaim_disowned", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(*new_owner == 19); + hld.reclaim_disowned(); // Manually veriified: without this, clang++ -fsanitize=address reports + // "detected memory leaks". + // NOLINTNEXTLINE(bugprone-unused-return-value) + (void) new_owner.release(); // Manually verified: without this, clang++ -fsanitize=address + // reports "attempting double-free". + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(new_owner.get() == nullptr); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+release_disowned", "[S]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE(poc::as_lvalue_ref(hld) == 19); + REQUIRE(*new_owner == 19); + hld.release_disowned(); + REQUIRE(!hld.has_pointee()); +} + +TEST_CASE("from_raw_ptr_take_ownership+disown+ensure_is_not_disowned", "[E]") { + const char *context = "test_case"; + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + hld.ensure_is_not_disowned(context); // Does not throw. + std::unique_ptr new_owner(hld.as_raw_ptr_unowned()); + hld.disown(); + REQUIRE_THROWS_WITH(hld.ensure_is_not_disowned(context), + "Holder was disowned already (test_case)."); +} + +TEST_CASE("from_unique_ptr+as_lvalue_ref", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership1", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto new_owner = std::unique_ptr(poc::as_raw_ptr_release_ownership(hld)); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr+as_raw_ptr_release_ownership2", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown use_count != 1 (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr1", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr2", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + auto shd_ptr = hld.as_shared_ptr(); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown use_count != 1 (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr+as_unique_ptr_with_deleter", "[E]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr+as_shared_ptr", "[S]") { + std::unique_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base", "[S]") { + std::unique_ptr orig_owner(new helpers::derived()); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr new_owner = poc::as_unique_ptr(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(new_owner->get() == 100); +} + +TEST_CASE("from_unique_ptr_derived+as_unique_ptr_base2", "[E]") { + std::unique_ptr> orig_owner( + new helpers::derived()); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH( + (poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_lvalue_ref", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr_with_std_function_deleter+as_lvalue_ref", "[S]") { + std::unique_ptr> orig_owner( + new int(19), [](const int *raw_ptr) { delete raw_ptr; }); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_raw_ptr_release_ownership", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown custom deleter (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter1", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::unique_ptr> new_owner + = poc::as_unique_ptr>(hld); + REQUIRE(!hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_unique_ptr_with_deleter2", "[E]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Incompatible unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr", "[S]") { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + REQUIRE(orig_owner.get() == nullptr); + std::shared_ptr new_owner = hld.as_shared_ptr(); + REQUIRE(hld.has_pointee()); + REQUIRE(*new_owner == 19); +} + +TEST_CASE("from_shared_ptr+as_lvalue_ref", "[S]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE(poc::as_lvalue_ref(hld) == 19); +} + +TEST_CASE("from_shared_ptr+as_raw_ptr_release_ownership", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH(poc::as_raw_ptr_release_ownership(hld), + "Cannot disown external shared_ptr (as_raw_ptr_release_ownership)."); +} + +TEST_CASE("from_shared_ptr+as_unique_ptr", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), + "Cannot disown external shared_ptr (as_unique_ptr)."); +} + +TEST_CASE("from_shared_ptr+as_unique_ptr_with_deleter", "[E]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE_THROWS_WITH((poc::as_unique_ptr>(hld)), + "Missing unique_ptr deleter (as_unique_ptr)."); +} + +TEST_CASE("from_shared_ptr+as_shared_ptr", "[S]") { + std::shared_ptr orig_owner(new int(19)); + auto hld = smart_holder::from_shared_ptr(orig_owner); + REQUIRE(*hld.as_shared_ptr() == 19); +} + +TEST_CASE("error_unpopulated_holder", "[E]") { + smart_holder hld; + REQUIRE_THROWS_WITH(poc::as_lvalue_ref(hld), "Unpopulated holder (as_lvalue_ref)."); +} + +TEST_CASE("error_disowned_holder", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + poc::as_unique_ptr(hld); + REQUIRE_THROWS_WITH(poc::as_lvalue_ref(hld), "Disowned holder (as_lvalue_ref)."); +} + +TEST_CASE("error_cannot_disown_nullptr", "[E]") { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + poc::as_unique_ptr(hld); + REQUIRE_THROWS_WITH(poc::as_unique_ptr(hld), "Cannot disown nullptr (as_unique_ptr)."); +} + +TEST_CASE("indestructible_int-from_raw_ptr_unowned+as_raw_ptr_unowned", "[S]") { + using zombie = helpers::indestructible_int; + // Using placement new instead of plain new, to not trigger leak sanitizer errors. + static std::aligned_storage::type memory_block[1]; + auto *value = new (memory_block) zombie(19); + auto hld = smart_holder::from_raw_ptr_unowned(value); + REQUIRE(hld.as_raw_ptr_unowned()->valu == 19); +} + +TEST_CASE("indestructible_int-from_raw_ptr_take_ownership", "[E]") { + helpers::indestructible_int *value = nullptr; + REQUIRE_THROWS_WITH(smart_holder::from_raw_ptr_take_ownership(value), + "Pointee is not destructible (from_raw_ptr_take_ownership)."); +} + +TEST_CASE("from_raw_ptr_take_ownership+as_shared_ptr-outliving_smart_holder", "[S]") { + // Exercises guarded_builtin_delete flag_ptr validity past destruction of smart_holder. + std::shared_ptr longer_living; + { + auto hld = smart_holder::from_raw_ptr_take_ownership(new int(19)); + longer_living = hld.as_shared_ptr(); + } + REQUIRE(*longer_living == 19); +} + +TEST_CASE("from_unique_ptr_with_deleter+as_shared_ptr-outliving_smart_holder", "[S]") { + // Exercises guarded_custom_deleter flag_ptr validity past destruction of smart_holder. + std::shared_ptr longer_living; + { + std::unique_ptr> orig_owner(new int(19)); + auto hld = smart_holder::from_unique_ptr(std::move(orig_owner)); + longer_living = hld.as_shared_ptr(); + } + REQUIRE(*longer_living == 19); +} diff --git a/tests/test_class.cpp b/tests/test_class.cpp index cb84c327a0..86c8d37017 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -105,6 +105,12 @@ TEST_SUBMODULE(class_, m) { .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); + // test_pass_unique_ptr + struct ToBeHeldByUniquePtr {}; + py::class_>(m, "ToBeHeldByUniquePtr") + .def(py::init<>()); + m.def("pass_unique_ptr", [](std::unique_ptr &&) {}); + // test_inheritance class Pet { public: @@ -227,11 +233,12 @@ TEST_SUBMODULE(class_, m) { m.def("mismatched_holder_1", []() { auto mod = py::module_::import("__main__"); py::class_>(mod, "MismatchBase1"); - py::class_(mod, "MismatchDerived1"); + py::class_, MismatchBase1>( + mod, "MismatchDerived1"); }); m.def("mismatched_holder_2", []() { auto mod = py::module_::import("__main__"); - py::class_(mod, "MismatchBase2"); + py::class_>(mod, "MismatchBase2"); py::class_, MismatchBase2>( mod, "MismatchDerived2"); }); @@ -625,8 +632,10 @@ CHECK_NOALIAS(8); CHECK_HOLDER(1, unique); CHECK_HOLDER(2, unique); CHECK_HOLDER(3, unique); +#ifndef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT CHECK_HOLDER(4, unique); CHECK_HOLDER(5, unique); +#endif CHECK_HOLDER(6, shared); CHECK_HOLDER(7, shared); CHECK_HOLDER(8, shared); diff --git a/tests/test_class.py b/tests/test_class.py index 01963d0122..b8102ec63b 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -49,6 +49,16 @@ def test_instance_new(): assert cstats.alive() == 0 +def test_pass_unique_ptr(): + obj = m.ToBeHeldByUniquePtr() + with pytest.raises(RuntimeError) as execinfo: + m.pass_unique_ptr(obj) + assert str(execinfo.value).startswith( + "Passing `std::unique_ptr` from Python to C++ requires `py::classh` (with T = " + ) + assert "ToBeHeldByUniquePtr" in str(execinfo.value) + + def test_type(): assert m.check_type(1) == m.DerivedClass1 with pytest.raises(RuntimeError) as execinfo: diff --git a/tests/test_class_sh_basic.cpp b/tests/test_class_sh_basic.cpp new file mode 100644 index 0000000000..ee973c2732 --- /dev/null +++ b/tests/test_class_sh_basic.cpp @@ -0,0 +1,249 @@ +#include + +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace class_sh_basic { + +struct atyp { // Short for "any type". + std::string mtxt; + atyp() : mtxt("DefaultConstructor") {} + explicit atyp(const std::string &mtxt_) : mtxt(mtxt_) {} + atyp(const atyp &other) { mtxt = other.mtxt + "_CpCtor"; } + atyp(atyp &&other) noexcept { mtxt = other.mtxt + "_MvCtor"; } +}; + +struct uconsumer { // unique_ptr consumer + std::unique_ptr held; + bool valid() const { return static_cast(held); } + + void pass_valu(std::unique_ptr obj) { held = std::move(obj); } + void pass_rref(std::unique_ptr &&obj) { held = std::move(obj); } + std::unique_ptr rtrn_valu() { return std::move(held); } + std::unique_ptr &rtrn_lref() { return held; } + const std::unique_ptr &rtrn_cref() const { return held; } +}; + +/// Custom deleter that is default constructible. +struct custom_deleter { + std::string trace_txt; + + custom_deleter() = default; + explicit custom_deleter(const std::string &trace_txt_) : trace_txt(trace_txt_) {} + + custom_deleter(const custom_deleter &other) { trace_txt = other.trace_txt + "_CpCtor"; } + + custom_deleter &operator=(const custom_deleter &rhs) { + trace_txt = rhs.trace_txt + "_CpLhs"; + return *this; + } + + custom_deleter(custom_deleter &&other) noexcept { + trace_txt = other.trace_txt + "_MvCtorTo"; + other.trace_txt += "_MvCtorFrom"; + } + + custom_deleter &operator=(custom_deleter &&rhs) noexcept { + trace_txt = rhs.trace_txt + "_MvLhs"; + rhs.trace_txt += "_MvRhs"; + return *this; + } + + void operator()(atyp *p) const { std::default_delete()(p); } + void operator()(const atyp *p) const { std::default_delete()(p); } +}; +static_assert(std::is_default_constructible::value, ""); + +/// Custom deleter that is not default constructible. +struct custom_deleter_nd : custom_deleter { + custom_deleter_nd() = delete; + explicit custom_deleter_nd(const std::string &trace_txt_) : custom_deleter(trace_txt_) {} +}; +static_assert(!std::is_default_constructible::value, ""); + +// clang-format off + +atyp rtrn_valu() { atyp obj{"rtrn_valu"}; return obj; } +atyp&& rtrn_rref() { static atyp obj; obj.mtxt = "rtrn_rref"; return std::move(obj); } +atyp const& rtrn_cref() { static atyp obj; obj.mtxt = "rtrn_cref"; return obj; } +atyp& rtrn_mref() { static atyp obj; obj.mtxt = "rtrn_mref"; return obj; } +atyp const* rtrn_cptr() { return new atyp{"rtrn_cptr"}; } +atyp* rtrn_mptr() { return new atyp{"rtrn_mptr"}; } + +std::string pass_valu(atyp obj) { return "pass_valu:" + obj.mtxt; } // NOLINT +std::string pass_cref(atyp const& obj) { return "pass_cref:" + obj.mtxt; } +std::string pass_mref(atyp& obj) { return "pass_mref:" + obj.mtxt; } +std::string pass_cptr(atyp const* obj) { return "pass_cptr:" + obj->mtxt; } +std::string pass_mptr(atyp* obj) { return "pass_mptr:" + obj->mtxt; } + +std::shared_ptr rtrn_shmp() { return std::make_shared("rtrn_shmp"); } +std::shared_ptr rtrn_shcp() { return std::shared_ptr(new atyp{"rtrn_shcp"}); } + +std::string pass_shmp(std::shared_ptr obj) { return "pass_shmp:" + obj->mtxt; } // NOLINT +std::string pass_shcp(std::shared_ptr obj) { return "pass_shcp:" + obj->mtxt; } // NOLINT + +std::unique_ptr rtrn_uqmp() { return std::unique_ptr(new atyp{"rtrn_uqmp"}); } +std::unique_ptr rtrn_uqcp() { return std::unique_ptr(new atyp{"rtrn_uqcp"}); } + +std::string pass_uqmp(std::unique_ptr obj) { return "pass_uqmp:" + obj->mtxt; } +std::string pass_uqcp(std::unique_ptr obj) { return "pass_uqcp:" + obj->mtxt; } + +struct sddm : std::default_delete {}; +struct sddc : std::default_delete {}; + +std::unique_ptr rtrn_udmp() { return std::unique_ptr(new atyp{"rtrn_udmp"}); } +std::unique_ptr rtrn_udcp() { return std::unique_ptr(new atyp{"rtrn_udcp"}); } + +std::string pass_udmp(std::unique_ptr obj) { return "pass_udmp:" + obj->mtxt; } +std::string pass_udcp(std::unique_ptr obj) { return "pass_udcp:" + obj->mtxt; } + +std::unique_ptr rtrn_udmp_del() { return std::unique_ptr(new atyp{"rtrn_udmp_del"}, custom_deleter{"udmp_deleter"}); } +std::unique_ptr rtrn_udcp_del() { return std::unique_ptr(new atyp{"rtrn_udcp_del"}, custom_deleter{"udcp_deleter"}); } + +std::string pass_udmp_del(std::unique_ptr obj) { return "pass_udmp_del:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } +std::string pass_udcp_del(std::unique_ptr obj) { return "pass_udcp_del:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } + +std::unique_ptr rtrn_udmp_del_nd() { return std::unique_ptr(new atyp{"rtrn_udmp_del_nd"}, custom_deleter_nd{"udmp_deleter_nd"}); } +std::unique_ptr rtrn_udcp_del_nd() { return std::unique_ptr(new atyp{"rtrn_udcp_del_nd"}, custom_deleter_nd{"udcp_deleter_nd"}); } + +std::string pass_udmp_del_nd(std::unique_ptr obj) { return "pass_udmp_del_nd:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } +std::string pass_udcp_del_nd(std::unique_ptr obj) { return "pass_udcp_del_nd:" + obj->mtxt + "," + obj.get_deleter().trace_txt; } + +// clang-format on + +// Helpers for testing. +std::string get_mtxt(atyp const &obj) { return obj.mtxt; } +std::ptrdiff_t get_ptr(atyp const &obj) { return reinterpret_cast(&obj); } + +std::unique_ptr unique_ptr_roundtrip(std::unique_ptr obj) { return obj; } + +std::string pass_unique_ptr_cref(const std::unique_ptr &obj) { return obj->mtxt; } + +const std::unique_ptr &rtrn_unique_ptr_cref(const std::string &mtxt) { + static std::unique_ptr obj{new atyp{"static_ctor_arg"}}; + if (!mtxt.empty()) { + obj->mtxt = mtxt; + } + return obj; +} + +const std::unique_ptr &unique_ptr_cref_roundtrip(const std::unique_ptr &obj) { + return obj; +} + +struct SharedPtrStash { + std::vector> stash; + void Add(const std::shared_ptr &obj) { stash.push_back(obj); } +}; + +class LocalUnusualOpRef : UnusualOpRef {}; // To avoid clashing with `py::class_`. +py::object CastUnusualOpRefConstRef(const LocalUnusualOpRef &cref) { return py::cast(cref); } +py::object CastUnusualOpRefMovable(LocalUnusualOpRef &&mvbl) { return py::cast(std::move(mvbl)); } + +TEST_SUBMODULE(class_sh_basic, m) { + namespace py = pybind11; + + py::classh(m, "atyp").def(py::init<>()).def(py::init([](const std::string &mtxt) { + atyp obj; + obj.mtxt = mtxt; + return obj; + })); + + m.def("rtrn_valu", rtrn_valu); + m.def("rtrn_rref", rtrn_rref); + m.def("rtrn_cref", rtrn_cref); + m.def("rtrn_mref", rtrn_mref); + m.def("rtrn_cptr", rtrn_cptr); + m.def("rtrn_mptr", rtrn_mptr); + + m.def("pass_valu", pass_valu); + m.def("pass_cref", pass_cref); + m.def("pass_mref", pass_mref); + m.def("pass_cptr", pass_cptr); + m.def("pass_mptr", pass_mptr); + + m.def("rtrn_shmp", rtrn_shmp); + m.def("rtrn_shcp", rtrn_shcp); + + m.def("pass_shmp", pass_shmp); + m.def("pass_shcp", pass_shcp); + + m.def("rtrn_uqmp", rtrn_uqmp); + m.def("rtrn_uqcp", rtrn_uqcp); + + m.def("pass_uqmp", pass_uqmp); + m.def("pass_uqcp", pass_uqcp); + + m.def("rtrn_udmp", rtrn_udmp); + m.def("rtrn_udcp", rtrn_udcp); + + m.def("pass_udmp", pass_udmp); + m.def("pass_udcp", pass_udcp); + + m.def("rtrn_udmp_del", rtrn_udmp_del); + m.def("rtrn_udcp_del", rtrn_udcp_del); + + m.def("pass_udmp_del", pass_udmp_del); + m.def("pass_udcp_del", pass_udcp_del); + + m.def("rtrn_udmp_del_nd", rtrn_udmp_del_nd); + m.def("rtrn_udcp_del_nd", rtrn_udcp_del_nd); + + m.def("pass_udmp_del_nd", pass_udmp_del_nd); + m.def("pass_udcp_del_nd", pass_udcp_del_nd); + + py::classh(m, "uconsumer") + .def(py::init<>()) + .def("valid", &uconsumer::valid) + .def("pass_valu", &uconsumer::pass_valu) + .def("pass_rref", &uconsumer::pass_rref) + .def("rtrn_valu", &uconsumer::rtrn_valu) + .def("rtrn_lref", &uconsumer::rtrn_lref) + .def("rtrn_cref", &uconsumer::rtrn_cref); + + // Helpers for testing. + // These require selected functions above to work first, as indicated: + m.def("get_mtxt", get_mtxt); // pass_cref + m.def("get_ptr", get_ptr); // pass_cref + + m.def("unique_ptr_roundtrip", unique_ptr_roundtrip); // pass_uqmp, rtrn_uqmp + + m.def("pass_unique_ptr_cref", pass_unique_ptr_cref); + m.def("rtrn_unique_ptr_cref", rtrn_unique_ptr_cref); + m.def("unique_ptr_cref_roundtrip", unique_ptr_cref_roundtrip); + + py::classh(m, "SharedPtrStash") + .def(py::init<>()) + .def("Add", &SharedPtrStash::Add, py::arg("obj")); + + m.def("py_type_handle_of_atyp", []() { + return py::type::handle_of(); // Exercises static_cast in this function. + }); + + // Checks for type names used as arguments + m.def("args_shared_ptr", [](std::shared_ptr p) { return p; }); + m.def("args_shared_ptr_const", [](std::shared_ptr p) { return p; }); + m.def("args_unique_ptr", [](std::unique_ptr p) { return p; }); + m.def("args_unique_ptr_const", [](std::unique_ptr p) { return p; }); + + // Make sure unique_ptr type caster accept automatic_reference return value policy. + m.def( + "rtrn_uq_automatic_reference", + []() { return std::unique_ptr(new atyp("rtrn_uq_automatic_reference")); }, + pybind11::return_value_policy::automatic_reference); + + m.def("pass_shared_ptr_ptr", [](std::shared_ptr *) {}); + + py::classh(m, "LocalUnusualOpRef"); + m.def("CallCastUnusualOpRefConstRef", + []() { return CastUnusualOpRefConstRef(LocalUnusualOpRef()); }); + m.def("CallCastUnusualOpRefMovable", + []() { return CastUnusualOpRefMovable(LocalUnusualOpRef()); }); +} + +} // namespace class_sh_basic +} // namespace pybind11_tests diff --git a/tests/test_class_sh_basic.py b/tests/test_class_sh_basic.py new file mode 100644 index 0000000000..f07253a5df --- /dev/null +++ b/tests/test_class_sh_basic.py @@ -0,0 +1,246 @@ +# Importing re before pytest after observing a PyPy CI flake when importing pytest first. +from __future__ import annotations + +import re + +import pytest + +from pybind11_tests import class_sh_basic as m + + +def test_atyp_constructors(): + obj = m.atyp() + assert obj.__class__.__name__ == "atyp" + obj = m.atyp("") + assert obj.__class__.__name__ == "atyp" + obj = m.atyp("txtm") + assert obj.__class__.__name__ == "atyp" + + +@pytest.mark.parametrize( + ("rtrn_f", "expected"), + [ + (m.rtrn_valu, "rtrn_valu(_MvCtor)*_MvCtor"), + (m.rtrn_rref, "rtrn_rref(_MvCtor)*_MvCtor"), + (m.rtrn_cref, "rtrn_cref(_MvCtor)*_CpCtor"), + (m.rtrn_mref, "rtrn_mref(_MvCtor)*_CpCtor"), + (m.rtrn_cptr, "rtrn_cptr"), + (m.rtrn_mptr, "rtrn_mptr"), + (m.rtrn_shmp, "rtrn_shmp"), + (m.rtrn_shcp, "rtrn_shcp"), + (m.rtrn_uqmp, "rtrn_uqmp"), + (m.rtrn_uqcp, "rtrn_uqcp"), + (m.rtrn_udmp, "rtrn_udmp"), + (m.rtrn_udcp, "rtrn_udcp"), + ], +) +def test_cast(rtrn_f, expected): + assert re.match(expected, m.get_mtxt(rtrn_f())) + + +@pytest.mark.parametrize( + ("pass_f", "mtxt", "expected"), + [ + (m.pass_valu, "Valu", "pass_valu:Valu(_MvCtor)*_CpCtor"), + (m.pass_cref, "Cref", "pass_cref:Cref(_MvCtor)*_MvCtor"), + (m.pass_mref, "Mref", "pass_mref:Mref(_MvCtor)*_MvCtor"), + (m.pass_cptr, "Cptr", "pass_cptr:Cptr(_MvCtor)*_MvCtor"), + (m.pass_mptr, "Mptr", "pass_mptr:Mptr(_MvCtor)*_MvCtor"), + (m.pass_shmp, "Shmp", "pass_shmp:Shmp(_MvCtor)*_MvCtor"), + (m.pass_shcp, "Shcp", "pass_shcp:Shcp(_MvCtor)*_MvCtor"), + (m.pass_uqmp, "Uqmp", "pass_uqmp:Uqmp(_MvCtor)*_MvCtor"), + (m.pass_uqcp, "Uqcp", "pass_uqcp:Uqcp(_MvCtor)*_MvCtor"), + ], +) +def test_load_with_mtxt(pass_f, mtxt, expected): + assert re.match(expected, pass_f(m.atyp(mtxt))) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "expected"), + [ + (m.pass_udmp, m.rtrn_udmp, "pass_udmp:rtrn_udmp"), + (m.pass_udcp, m.rtrn_udcp, "pass_udcp:rtrn_udcp"), + ], +) +def test_load_with_rtrn_f(pass_f, rtrn_f, expected): + assert pass_f(rtrn_f()) == expected + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "regex_expected"), + [ + ( + m.pass_udmp_del, + m.rtrn_udmp_del, + "pass_udmp_del:rtrn_udmp_del,udmp_deleter(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udcp_del, + m.rtrn_udcp_del, + "pass_udcp_del:rtrn_udcp_del,udcp_deleter(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udmp_del_nd, + m.rtrn_udmp_del_nd, + "pass_udmp_del_nd:rtrn_udmp_del_nd,udmp_deleter_nd(_MvCtorTo)*_MvCtorTo", + ), + ( + m.pass_udcp_del_nd, + m.rtrn_udcp_del_nd, + "pass_udcp_del_nd:rtrn_udcp_del_nd,udcp_deleter_nd(_MvCtorTo)*_MvCtorTo", + ), + ], +) +def test_deleter_roundtrip(pass_f, rtrn_f, regex_expected): + assert re.match(regex_expected, pass_f(rtrn_f())) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "expected"), + [ + (m.pass_uqmp, m.rtrn_uqmp, "pass_uqmp:rtrn_uqmp"), + (m.pass_uqcp, m.rtrn_uqcp, "pass_uqcp:rtrn_uqcp"), + (m.pass_udmp, m.rtrn_udmp, "pass_udmp:rtrn_udmp"), + (m.pass_udcp, m.rtrn_udcp, "pass_udcp:rtrn_udcp"), + ], +) +def test_pass_unique_ptr_disowns(pass_f, rtrn_f, expected): + obj = rtrn_f() + assert pass_f(obj) == expected + with pytest.raises(ValueError) as exc_info: + pass_f(obj) + assert str(exc_info.value) == ( + "Missing value for wrapped C++ type" + + " `pybind11_tests::class_sh_basic::atyp`:" + + " Python instance was disowned." + ) + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f"), + [ + (m.pass_uqmp, m.rtrn_uqmp), + (m.pass_uqcp, m.rtrn_uqcp), + (m.pass_udmp, m.rtrn_udmp), + (m.pass_udcp, m.rtrn_udcp), + ], +) +def test_cannot_disown_use_count_ne_1(pass_f, rtrn_f): + obj = rtrn_f() + stash = m.SharedPtrStash() + stash.Add(obj) + with pytest.raises(ValueError) as exc_info: + pass_f(obj) + assert str(exc_info.value) == ("Cannot disown use_count != 1 (load_as_unique_ptr).") + + +def test_unique_ptr_roundtrip(num_round_trips=1000): + # Multiple roundtrips to stress-test instance registration/deregistration. + recycled = m.atyp("passenger") + for _ in range(num_round_trips): + id_orig = id(recycled) + recycled = m.unique_ptr_roundtrip(recycled) + assert re.match("passenger(_MvCtor)*_MvCtor", m.get_mtxt(recycled)) + id_rtrn = id(recycled) + # Ensure the returned object is a different Python instance. + assert id_rtrn != id_orig + id_orig = id_rtrn + + +def test_pass_unique_ptr_cref(): + obj = m.atyp("ctor_arg") + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.get_mtxt(obj)) + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.pass_unique_ptr_cref(obj)) + assert re.match("ctor_arg(_MvCtor)*_MvCtor", m.get_mtxt(obj)) + + +def test_rtrn_unique_ptr_cref(): + obj0 = m.rtrn_unique_ptr_cref("") + assert m.get_mtxt(obj0) == "static_ctor_arg" + obj1 = m.rtrn_unique_ptr_cref("passed_mtxt_1") + assert m.get_mtxt(obj1) == "passed_mtxt_1" + assert m.get_mtxt(obj0) == "passed_mtxt_1" + assert obj0 is obj1 + + +def test_unique_ptr_cref_roundtrip(num_round_trips=1000): + # Multiple roundtrips to stress-test implementation. + orig = m.atyp("passenger") + mtxt_orig = m.get_mtxt(orig) + recycled = orig + for _ in range(num_round_trips): + recycled = m.unique_ptr_cref_roundtrip(recycled) + assert recycled is orig + assert m.get_mtxt(recycled) == mtxt_orig + + +@pytest.mark.parametrize( + ("pass_f", "rtrn_f", "moved_out", "moved_in"), + [ + (m.uconsumer.pass_valu, m.uconsumer.rtrn_valu, True, True), + (m.uconsumer.pass_rref, m.uconsumer.rtrn_valu, True, True), + (m.uconsumer.pass_valu, m.uconsumer.rtrn_lref, True, False), + (m.uconsumer.pass_valu, m.uconsumer.rtrn_cref, True, False), + ], +) +def test_unique_ptr_consumer_roundtrip(pass_f, rtrn_f, moved_out, moved_in): + c = m.uconsumer() + assert not c.valid() + recycled = m.atyp("passenger") + mtxt_orig = m.get_mtxt(recycled) + assert re.match("passenger_(MvCtor){1,2}", mtxt_orig) + + pass_f(c, recycled) + if moved_out: + with pytest.raises(ValueError) as excinfo: + m.get_mtxt(recycled) + assert "Python instance was disowned" in str(excinfo.value) + + recycled = rtrn_f(c) + assert c.valid() != moved_in + assert m.get_mtxt(recycled) == mtxt_orig + + +def test_py_type_handle_of_atyp(): + obj = m.py_type_handle_of_atyp() + assert obj.__class__.__name__ == "pybind11_type" + + +def test_function_signatures(doc): + assert ( + doc(m.args_shared_ptr) + == "args_shared_ptr(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_shared_ptr_const) + == "args_shared_ptr_const(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_unique_ptr) + == "args_unique_ptr(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + assert ( + doc(m.args_unique_ptr_const) + == "args_unique_ptr_const(arg0: m.class_sh_basic.atyp) -> m.class_sh_basic.atyp" + ) + + +def test_unique_ptr_return_value_policy_automatic_reference(): + assert m.get_mtxt(m.rtrn_uq_automatic_reference()) == "rtrn_uq_automatic_reference" + + +def test_pass_shared_ptr_ptr(): + obj = m.atyp() + with pytest.raises(RuntimeError) as excinfo: + m.pass_shared_ptr_ptr(obj) + assert str(excinfo.value) == ( + "Passing `std::shared_ptr *` from Python to C++ is not supported" + " (inherently unsafe)." + ) + + +def test_unusual_op_ref(): + # Merely to test that this still exists and built successfully. + assert m.CallCastUnusualOpRefConstRef().__class__.__name__ == "LocalUnusualOpRef" + assert m.CallCastUnusualOpRefMovable().__class__.__name__ == "LocalUnusualOpRef" diff --git a/tests/test_class_sh_disowning.cpp b/tests/test_class_sh_disowning.cpp new file mode 100644 index 0000000000..97c95287be --- /dev/null +++ b/tests/test_class_sh_disowning.cpp @@ -0,0 +1,43 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_disowning { + +template // Using int as a trick to easily generate a series of types. +struct Atype { + int val = 0; + explicit Atype(int val_) : val{val_} {} + int get() const { return val * 10 + SerNo; } +}; + +int same_twice(std::unique_ptr> at1a, std::unique_ptr> at1b) { + return at1a->get() * 100 + at1b->get() * 10; +} + +int mixed(std::unique_ptr> at1, std::unique_ptr> at2) { + return at1->get() * 200 + at2->get() * 20; +} + +int overloaded(std::unique_ptr> at1, int i) { return at1->get() * 30 + i; } +int overloaded(std::unique_ptr> at2, int i) { return at2->get() * 40 + i; } + +} // namespace class_sh_disowning +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_disowning, m) { + using namespace pybind11_tests::class_sh_disowning; + + py::classh>(m, "Atype1").def(py::init()).def("get", &Atype<1>::get); + py::classh>(m, "Atype2").def(py::init()).def("get", &Atype<2>::get); + + m.def("same_twice", same_twice); + + m.def("mixed", mixed); + + m.def("overloaded", (int (*)(std::unique_ptr>, int)) &overloaded); + m.def("overloaded", (int (*)(std::unique_ptr>, int)) &overloaded); +} diff --git a/tests/test_class_sh_disowning.py b/tests/test_class_sh_disowning.py new file mode 100644 index 0000000000..b9e648999f --- /dev/null +++ b/tests/test_class_sh_disowning.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_disowning as m + + +def is_disowned(obj): + try: + obj.get() + except ValueError: + return True + return False + + +def test_same_twice(): + while True: + obj1a = m.Atype1(57) + obj1b = m.Atype1(62) + assert m.same_twice(obj1a, obj1b) == (57 * 10 + 1) * 100 + (62 * 10 + 1) * 10 + assert is_disowned(obj1a) + assert is_disowned(obj1b) + obj1c = m.Atype1(0) + with pytest.raises(ValueError): + # Disowning works for one argument, but not both. + m.same_twice(obj1c, obj1c) + assert is_disowned(obj1c) + return # Comment out for manual leak checking (use `top` command). + + +def test_mixed(): + first_pass = True + while True: + obj1a = m.Atype1(90) + obj2a = m.Atype2(25) + assert m.mixed(obj1a, obj2a) == (90 * 10 + 1) * 200 + (25 * 10 + 2) * 20 + assert is_disowned(obj1a) + assert is_disowned(obj2a) + + # The C++ order of evaluation of function arguments is (unfortunately) unspecified: + # https://en.cppreference.com/w/cpp/language/eval_order + # Read on. + obj1b = m.Atype1(0) + with pytest.raises(ValueError): + # If the 1st argument is evaluated first, obj1b is disowned before the conversion for + # the already disowned obj2a fails as expected. + m.mixed(obj1b, obj2a) + obj2b = m.Atype2(0) + with pytest.raises(ValueError): + # If the 2nd argument is evaluated first, obj2b is disowned before the conversion for + # the already disowned obj1a fails as expected. + m.mixed(obj1a, obj2b) + + # Either obj1b or obj2b was disowned in the expected failed m.mixed() calls above, but not + # both. + is_disowned_results = (is_disowned(obj1b), is_disowned(obj2b)) + assert is_disowned_results.count(True) == 1 + if first_pass: + first_pass = False + ix = is_disowned_results.index(True) + 1 + print(f"\nC++ function argument {ix} is evaluated first.") + + return # Comment out for manual leak checking (use `top` command). + + +def test_overloaded(): + while True: + obj1 = m.Atype1(81) + obj2 = m.Atype2(60) + with pytest.raises(TypeError): + m.overloaded(obj1, "NotInt") + assert obj1.get() == 81 * 10 + 1 # Not disowned. + assert m.overloaded(obj1, 3) == (81 * 10 + 1) * 30 + 3 + with pytest.raises(TypeError): + m.overloaded(obj2, "NotInt") + assert obj2.get() == 60 * 10 + 2 # Not disowned. + assert m.overloaded(obj2, 2) == (60 * 10 + 2) * 40 + 2 + return # Comment out for manual leak checking (use `top` command). diff --git a/tests/test_class_sh_disowning_mi.cpp b/tests/test_class_sh_disowning_mi.cpp new file mode 100644 index 0000000000..8b96672536 --- /dev/null +++ b/tests/test_class_sh_disowning_mi.cpp @@ -0,0 +1,87 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_disowning_mi { + +// Diamond inheritance (copied from test_multiple_inheritance.cpp). +struct B { + int val_b = 10; + B() = default; + B(const B &) = default; + virtual ~B() = default; +}; + +struct C0 : public virtual B { + int val_c0 = 20; +}; + +struct C1 : public virtual B { + int val_c1 = 21; +}; + +struct D : public C0, public C1 { + int val_d = 30; +}; + +void disown_b(std::unique_ptr) {} + +// test_multiple_inheritance_python +struct Base1 { + explicit Base1(int i) : i(i) {} + int foo() const { return i; } + int i; +}; + +struct Base2 { + explicit Base2(int j) : j(j) {} + int bar() const { return j; } + int j; +}; + +int disown_base1(std::unique_ptr b1) { return b1->i * 2000 + 1; } +int disown_base2(std::unique_ptr b2) { return b2->j * 2000 + 2; } + +} // namespace class_sh_disowning_mi +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_disowning_mi, m) { + using namespace pybind11_tests::class_sh_disowning_mi; + + py::classh(m, "B") + .def(py::init<>()) + .def_readonly("val_b", &D::val_b) + .def("b", [](B *self) { return self; }) + .def("get", [](const B &self) { return self.val_b; }); + + py::classh(m, "C0") + .def(py::init<>()) + .def_readonly("val_c0", &D::val_c0) + .def("c0", [](C0 *self) { return self; }) + .def("get", [](const C0 &self) { return self.val_b * 100 + self.val_c0; }); + + py::classh(m, "C1") + .def(py::init<>()) + .def_readonly("val_c1", &D::val_c1) + .def("c1", [](C1 *self) { return self; }) + .def("get", [](const C1 &self) { return self.val_b * 100 + self.val_c1; }); + + py::classh(m, "D") + .def(py::init<>()) + .def_readonly("val_d", &D::val_d) + .def("d", [](D *self) { return self; }) + .def("get", [](const D &self) { + return self.val_b * 1000000 + self.val_c0 * 10000 + self.val_c1 * 100 + self.val_d; + }); + + m.def("disown_b", disown_b); + + // test_multiple_inheritance_python + py::classh(m, "Base1").def(py::init()).def("foo", &Base1::foo); + py::classh(m, "Base2").def(py::init()).def("bar", &Base2::bar); + m.def("disown_base1", disown_base1); + m.def("disown_base2", disown_base2); +} diff --git a/tests/test_class_sh_disowning_mi.py b/tests/test_class_sh_disowning_mi.py new file mode 100644 index 0000000000..4a4beecce1 --- /dev/null +++ b/tests/test_class_sh_disowning_mi.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import pytest + +import env # noqa: F401 +from pybind11_tests import class_sh_disowning_mi as m + + +def test_diamond_inheritance(): + # Very similar to test_multiple_inheritance.py:test_diamond_inheritance. + d = m.D() + assert d is d.d() + assert d is d.c0() + assert d is d.c1() + assert d is d.b() + assert d is d.c0().b() + assert d is d.c1().b() + assert d is d.c0().c1().b().c0().b() + + +def is_disowned(callable_method): + try: + callable_method() + except ValueError as e: + assert "Python instance was disowned" in str(e) # noqa: PT017 + return True + return False + + +def test_disown_b(): + b = m.B() + assert b.get() == 10 + m.disown_b(b) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["c0", "b"]) +def test_disown_c0(var_to_disown): + c0 = m.C0() + assert c0.get() == 1020 + b = c0.b() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(c0.get) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["c1", "b"]) +def test_disown_c1(var_to_disown): + c1 = m.C1() + assert c1.get() == 1021 + b = c1.b() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(c1.get) + assert is_disowned(b.get) + + +@pytest.mark.parametrize("var_to_disown", ["d", "c1", "c0", "b"]) +def test_disown_d(var_to_disown): + d = m.D() + assert d.get() == 10202130 + b = d.b() + c0 = d.c0() + c1 = d.c1() + m.disown_b(locals()[var_to_disown]) + assert is_disowned(d.get) + assert is_disowned(c1.get) + assert is_disowned(c0.get) + assert is_disowned(b.get) + + +# Based on test_multiple_inheritance.py:test_multiple_inheritance_python. +class MI1(m.Base1, m.Base2): + def __init__(self, i, j): + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class B1: + def v(self): + return 1 + + +class MI2(B1, m.Base1, m.Base2): + def __init__(self, i, j): + B1.__init__(self) + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class MI3(MI2): + def __init__(self, i, j): + MI2.__init__(self, i, j) + + +class MI4(MI3, m.Base2): + def __init__(self, i, j): + MI3.__init__(self, i, j) + # This should be ignored (Base2 is already initialized via MI2): + m.Base2.__init__(self, i + 100) + + +class MI5(m.Base2, B1, m.Base1): + def __init__(self, i, j): + B1.__init__(self) + m.Base1.__init__(self, i) + m.Base2.__init__(self, j) + + +class MI6(m.Base2, B1): + def __init__(self, i): + m.Base2.__init__(self, i) + B1.__init__(self) + + +class B2(B1): + def v(self): + return 2 + + +class B3: + def v(self): + return 3 + + +class B4(B3, B2): + def v(self): + return 4 + + +class MI7(B4, MI6): + def __init__(self, i): + B4.__init__(self) + MI6.__init__(self, i) + + +class MI8(MI6, B3): + def __init__(self, i): + MI6.__init__(self, i) + B3.__init__(self) + + +class MI8b(B3, MI6): + def __init__(self, i): + B3.__init__(self) + MI6.__init__(self, i) + + +@pytest.mark.xfail("env.PYPY") +def test_multiple_inheritance_python(): + # Based on test_multiple_inheritance.py:test_multiple_inheritance_python. + # Exercises values_and_holders with 2 value_and_holder instances. + + mi1 = MI1(1, 2) + assert mi1.foo() == 1 + assert mi1.bar() == 2 + + mi2 = MI2(3, 4) + assert mi2.v() == 1 + assert mi2.foo() == 3 + assert mi2.bar() == 4 + + mi3 = MI3(5, 6) + assert mi3.v() == 1 + assert mi3.foo() == 5 + assert mi3.bar() == 6 + + mi4 = MI4(7, 8) + assert mi4.v() == 1 + assert mi4.foo() == 7 + assert mi4.bar() == 8 + + mi5 = MI5(10, 11) + assert mi5.v() == 1 + assert mi5.foo() == 10 + assert mi5.bar() == 11 + + mi6 = MI6(12) + assert mi6.v() == 1 + assert mi6.bar() == 12 + + mi7 = MI7(13) + assert mi7.v() == 4 + assert mi7.bar() == 13 + + mi8 = MI8(14) + assert mi8.v() == 1 + assert mi8.bar() == 14 + + mi8b = MI8b(15) + assert mi8b.v() == 3 + assert mi8b.bar() == 15 + + +DISOWN_CLS_I_J_V_LIST = [ + (MI1, 1, 2, None), + (MI2, 3, 4, 1), + (MI3, 5, 6, 1), + (MI4, 7, 8, 1), + (MI5, 10, 11, 1), +] + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize(("cls", "i", "j", "v"), DISOWN_CLS_I_J_V_LIST) +def test_disown_base1_first(cls, i, j, v): + obj = cls(i, j) + assert obj.foo() == i + assert m.disown_base1(obj) == 2000 * i + 1 + assert is_disowned(obj.foo) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + if v is not None: + assert obj.v() == v + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize(("cls", "i", "j", "v"), DISOWN_CLS_I_J_V_LIST) +def test_disown_base2_first(cls, i, j, v): + obj = cls(i, j) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + assert obj.foo() == i + assert m.disown_base1(obj) == 2000 * i + 1 + assert is_disowned(obj.foo) + if v is not None: + assert obj.v() == v + + +@pytest.mark.xfail("env.PYPY", strict=False) +@pytest.mark.parametrize( + ("cls", "j", "v"), + [ + (MI6, 12, 1), + (MI7, 13, 4), + (MI8, 14, 1), + (MI8b, 15, 3), + ], +) +def test_disown_base2(cls, j, v): + obj = cls(j) + assert obj.bar() == j + assert m.disown_base2(obj) == 2000 * j + 2 + assert is_disowned(obj.bar) + assert obj.v() == v diff --git a/tests/test_class_sh_factory_constructors.cpp b/tests/test_class_sh_factory_constructors.cpp new file mode 100644 index 0000000000..3499eaea5d --- /dev/null +++ b/tests/test_class_sh_factory_constructors.cpp @@ -0,0 +1,166 @@ +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace pybind11_tests { +namespace class_sh_factory_constructors { + +template // Using int as a trick to easily generate a series of types. +struct atyp { // Short for "any type". + std::string mtxt; +}; + +template +std::string get_mtxt(const T &obj) { + return obj.mtxt; +} + +using atyp_valu = atyp<0x0>; +using atyp_rref = atyp<0x1>; +using atyp_cref = atyp<0x2>; +using atyp_mref = atyp<0x3>; +using atyp_cptr = atyp<0x4>; +using atyp_mptr = atyp<0x5>; +using atyp_shmp = atyp<0x6>; +using atyp_shcp = atyp<0x7>; +using atyp_uqmp = atyp<0x8>; +using atyp_uqcp = atyp<0x9>; +using atyp_udmp = atyp<0xA>; +using atyp_udcp = atyp<0xB>; + +// clang-format off + +atyp_valu rtrn_valu() { atyp_valu obj{"Valu"}; return obj; } +atyp_rref&& rtrn_rref() { static atyp_rref obj; obj.mtxt = "Rref"; return std::move(obj); } +atyp_cref const& rtrn_cref() { static atyp_cref obj; obj.mtxt = "Cref"; return obj; } +atyp_mref& rtrn_mref() { static atyp_mref obj; obj.mtxt = "Mref"; return obj; } +atyp_cptr const* rtrn_cptr() { return new atyp_cptr{"Cptr"}; } +atyp_mptr* rtrn_mptr() { return new atyp_mptr{"Mptr"}; } + +std::shared_ptr rtrn_shmp() { return std::make_shared(atyp_shmp{"Shmp"}); } +std::shared_ptr rtrn_shcp() { return std::shared_ptr(new atyp_shcp{"Shcp"}); } + +std::unique_ptr rtrn_uqmp() { return std::unique_ptr(new atyp_uqmp{"Uqmp"}); } +std::unique_ptr rtrn_uqcp() { return std::unique_ptr(new atyp_uqcp{"Uqcp"}); } + +struct sddm : std::default_delete {}; +struct sddc : std::default_delete {}; + +std::unique_ptr rtrn_udmp() { return std::unique_ptr(new atyp_udmp{"Udmp"}); } +std::unique_ptr rtrn_udcp() { return std::unique_ptr(new atyp_udcp{"Udcp"}); } + +// clang-format on + +// Minimalistic approach to achieve full coverage of construct() overloads for constructing +// smart_holder from unique_ptr and shared_ptr returns. +struct with_alias { + int val = 0; + virtual ~with_alias() = default; + // Some compilers complain about implicitly defined versions of some of the following: + with_alias() = default; + with_alias(const with_alias &) = default; + with_alias(with_alias &&) = default; + with_alias &operator=(const with_alias &) = default; + with_alias &operator=(with_alias &&) = default; +}; +struct with_alias_alias : with_alias {}; +struct sddwaa : std::default_delete {}; + +} // namespace class_sh_factory_constructors +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_factory_constructors, m) { + using namespace pybind11_tests::class_sh_factory_constructors; + + py::classh(m, "atyp_valu") + .def(py::init(&rtrn_valu)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_rref") + .def(py::init(&rtrn_rref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_cref") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_cref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_mref") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_mref)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_cptr") + // class_: ... must return a compatible ... + // classh: ... must return a compatible ... + // .def(py::init(&rtrn_cptr)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_mptr") + .def(py::init(&rtrn_mptr)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_shmp") + .def(py::init(&rtrn_shmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_shcp") + // py::class_>(m, "atyp_shcp") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_shcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_uqmp") + .def(py::init(&rtrn_uqmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_uqcp") + // class_: ... cannot pass object of non-trivial type ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_uqcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_udmp") + .def(py::init(&rtrn_udmp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "atyp_udcp") + // py::class_>(m, "atyp_udcp") + // class_: ... must return a compatible ... + // classh: ... cannot pass object of non-trivial type ... + // .def(py::init(&rtrn_udcp)) + .def("get_mtxt", get_mtxt); + + py::classh(m, "with_alias") + .def_readonly("val", &with_alias::val) + .def(py::init([](int i) { + auto p = std::unique_ptr(new with_alias_alias); + p->val = i * 100; + return p; + })) + .def(py::init([](int i, int j) { + auto p = std::unique_ptr(new with_alias_alias); + p->val = i * 100 + j * 10; + return p; + })) + .def(py::init([](int i, int j, int k) { + auto p = std::make_shared(); + p->val = i * 100 + j * 10 + k; + return p; + })) + .def(py::init( + [](int, int, int, int) { return std::unique_ptr(new with_alias); }, + [](int, int, int, int) { + return std::unique_ptr(new with_alias); // Invalid alias factory. + })) + .def(py::init([](int, int, int, int, int) { return std::make_shared(); }, + [](int, int, int, int, int) { + return std::make_shared(); // Invalid alias factory. + })); +} diff --git a/tests/test_class_sh_factory_constructors.py b/tests/test_class_sh_factory_constructors.py new file mode 100644 index 0000000000..5d45db6fd5 --- /dev/null +++ b/tests/test_class_sh_factory_constructors.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_factory_constructors as m + + +def test_atyp_factories(): + assert m.atyp_valu().get_mtxt() == "Valu" + assert m.atyp_rref().get_mtxt() == "Rref" + # sert m.atyp_cref().get_mtxt() == "Cref" + # sert m.atyp_mref().get_mtxt() == "Mref" + # sert m.atyp_cptr().get_mtxt() == "Cptr" + assert m.atyp_mptr().get_mtxt() == "Mptr" + assert m.atyp_shmp().get_mtxt() == "Shmp" + # sert m.atyp_shcp().get_mtxt() == "Shcp" + assert m.atyp_uqmp().get_mtxt() == "Uqmp" + # sert m.atyp_uqcp().get_mtxt() == "Uqcp" + assert m.atyp_udmp().get_mtxt() == "Udmp" + # sert m.atyp_udcp().get_mtxt() == "Udcp" + + +@pytest.mark.parametrize( + ("init_args", "expected"), + [ + ((3,), 300), + ((5, 7), 570), + ((9, 11, 13), 1023), + ], +) +def test_with_alias_success(init_args, expected): + assert m.with_alias(*init_args).val == expected + + +@pytest.mark.parametrize( + ("num_init_args", "smart_ptr"), + [ + (4, "std::unique_ptr"), + (5, "std::shared_ptr"), + ], +) +def test_with_alias_invalid(num_init_args, smart_ptr): + class PyDrvdWithAlias(m.with_alias): + pass + + with pytest.raises(TypeError) as excinfo: + PyDrvdWithAlias(*((0,) * num_init_args)) + assert ( + str(excinfo.value) + == "pybind11::init(): construction failed: returned " + + smart_ptr + + " pointee is not an alias instance" + ) diff --git a/tests/test_class_sh_inheritance.cpp b/tests/test_class_sh_inheritance.cpp new file mode 100644 index 0000000000..bb46fb92e0 --- /dev/null +++ b/tests/test_class_sh_inheritance.cpp @@ -0,0 +1,92 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_inheritance { + +template +struct base_template { + base_template() : base_id(Id) {} + virtual ~base_template() = default; + virtual int id() const { return base_id; } + int base_id; + + // Some compilers complain about implicitly defined versions of some of the following: + base_template(const base_template &) = default; + base_template(base_template &&) noexcept = default; + base_template &operator=(const base_template &) = default; + base_template &operator=(base_template &&) noexcept = default; +}; + +using base = base_template<100>; + +struct drvd : base { + int id() const override { return 2 * base_id; } +}; + +// clang-format off +inline drvd *rtrn_mptr_drvd() { return new drvd; } +inline base *rtrn_mptr_drvd_up_cast() { return new drvd; } + +inline int pass_cptr_base(base const *b) { return b->id() + 11; } +inline int pass_cptr_drvd(drvd const *d) { return d->id() + 12; } + +inline std::shared_ptr rtrn_shmp_drvd() { return std::make_shared(); } +inline std::shared_ptr rtrn_shmp_drvd_up_cast() { return std::make_shared(); } + +inline int pass_shcp_base(const std::shared_ptr& b) { return b->id() + 21; } +inline int pass_shcp_drvd(const std::shared_ptr& d) { return d->id() + 22; } +// clang-format on + +using base1 = base_template<110>; +using base2 = base_template<120>; + +// Not reusing base here because it would interfere with the single-inheritance test. +struct drvd2 : base1, base2 { + int id() const override { return 3 * base1::base_id + 4 * base2::base_id; } +}; + +// clang-format off +inline drvd2 *rtrn_mptr_drvd2() { return new drvd2; } +inline base1 *rtrn_mptr_drvd2_up_cast1() { return new drvd2; } +inline base2 *rtrn_mptr_drvd2_up_cast2() { return new drvd2; } + +inline int pass_cptr_base1(base1 const *b) { return b->id() + 21; } +inline int pass_cptr_base2(base2 const *b) { return b->id() + 22; } +inline int pass_cptr_drvd2(drvd2 const *d) { return d->id() + 23; } +// clang-format on + +TEST_SUBMODULE(class_sh_inheritance, m) { + py::classh(m, "base"); + py::classh(m, "drvd"); + + auto rvto = py::return_value_policy::take_ownership; + + m.def("rtrn_mptr_drvd", rtrn_mptr_drvd, rvto); + m.def("rtrn_mptr_drvd_up_cast", rtrn_mptr_drvd_up_cast, rvto); + m.def("pass_cptr_base", pass_cptr_base); + m.def("pass_cptr_drvd", pass_cptr_drvd); + + m.def("rtrn_shmp_drvd", rtrn_shmp_drvd); + m.def("rtrn_shmp_drvd_up_cast", rtrn_shmp_drvd_up_cast); + m.def("pass_shcp_base", pass_shcp_base); + m.def("pass_shcp_drvd", pass_shcp_drvd); + + // __init__ needed for Python inheritance. + py::classh(m, "base1").def(py::init<>()); + py::classh(m, "base2").def(py::init<>()); + py::classh(m, "drvd2"); + + m.def("rtrn_mptr_drvd2", rtrn_mptr_drvd2, rvto); + m.def("rtrn_mptr_drvd2_up_cast1", rtrn_mptr_drvd2_up_cast1, rvto); + m.def("rtrn_mptr_drvd2_up_cast2", rtrn_mptr_drvd2_up_cast2, rvto); + m.def("pass_cptr_base1", pass_cptr_base1); + m.def("pass_cptr_base2", pass_cptr_base2); + m.def("pass_cptr_drvd2", pass_cptr_drvd2); +} + +} // namespace class_sh_inheritance +} // namespace pybind11_tests diff --git a/tests/test_class_sh_inheritance.py b/tests/test_class_sh_inheritance.py new file mode 100644 index 0000000000..cd9d6f47e2 --- /dev/null +++ b/tests/test_class_sh_inheritance.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_inheritance as m + + +def test_rtrn_mptr_drvd_pass_cptr_base(): + d = m.rtrn_mptr_drvd() + i = m.pass_cptr_base(d) # load_impl Case 2a + assert i == 2 * 100 + 11 + + +def test_rtrn_shmp_drvd_pass_shcp_base(): + d = m.rtrn_shmp_drvd() + i = m.pass_shcp_base(d) # load_impl Case 2a + assert i == 2 * 100 + 21 + + +def test_rtrn_mptr_drvd_up_cast_pass_cptr_drvd(): + b = m.rtrn_mptr_drvd_up_cast() + # the base return is down-cast immediately. + assert b.__class__.__name__ == "drvd" + i = m.pass_cptr_drvd(b) + assert i == 2 * 100 + 12 + + +def test_rtrn_shmp_drvd_up_cast_pass_shcp_drvd(): + b = m.rtrn_shmp_drvd_up_cast() + # the base return is down-cast immediately. + assert b.__class__.__name__ == "drvd" + i = m.pass_shcp_drvd(b) + assert i == 2 * 100 + 22 + + +def test_rtrn_mptr_drvd2_pass_cptr_bases(): + d = m.rtrn_mptr_drvd2() + i1 = m.pass_cptr_base1(d) # load_impl Case 2c + assert i1 == 3 * 110 + 4 * 120 + 21 + i2 = m.pass_cptr_base2(d) + assert i2 == 3 * 110 + 4 * 120 + 22 + + +def test_rtrn_mptr_drvd2_up_casts_pass_cptr_drvd2(): + b1 = m.rtrn_mptr_drvd2_up_cast1() + assert b1.__class__.__name__ == "drvd2" + i1 = m.pass_cptr_drvd2(b1) + assert i1 == 3 * 110 + 4 * 120 + 23 + b2 = m.rtrn_mptr_drvd2_up_cast2() + assert b2.__class__.__name__ == "drvd2" + i2 = m.pass_cptr_drvd2(b2) + assert i2 == 3 * 110 + 4 * 120 + 23 + + +def test_python_drvd2(): + class Drvd2(m.base1, m.base2): + def __init__(self): + m.base1.__init__(self) + m.base2.__init__(self) + + d = Drvd2() + i1 = m.pass_cptr_base1(d) # load_impl Case 2b + assert i1 == 110 + 21 + i2 = m.pass_cptr_base2(d) + assert i2 == 120 + 22 diff --git a/tests/test_class_sh_mi_thunks.cpp b/tests/test_class_sh_mi_thunks.cpp new file mode 100644 index 0000000000..244ccbfacb --- /dev/null +++ b/tests/test_class_sh_mi_thunks.cpp @@ -0,0 +1,96 @@ +#include +#include + +#include "pybind11_tests.h" + +#include +#include +#include + +namespace test_class_sh_mi_thunks { + +// For general background: https://shaharmike.com/cpp/vtable-part2/ +// C++ vtables - Part 2 - Multiple Inheritance +// ... the compiler creates a 'thunk' method that corrects `this` ... + +struct Base0 { + virtual ~Base0() = default; + Base0() = default; + Base0(const Base0 &) = delete; +}; + +struct Base1 { + virtual ~Base1() = default; + // Using `vector` here because it is known to make this test very sensitive to bugs. + std::vector vec = {1, 2, 3, 4, 5}; + Base1() = default; + Base1(const Base1 &) = delete; +}; + +struct Derived : Base1, Base0 { + ~Derived() override = default; + Derived() = default; + Derived(const Derived &) = delete; +}; + +} // namespace test_class_sh_mi_thunks + +TEST_SUBMODULE(class_sh_mi_thunks, m) { + using namespace test_class_sh_mi_thunks; + + m.def("ptrdiff_drvd_base0", []() { + auto drvd = std::unique_ptr(new Derived); + auto *base0 = dynamic_cast(drvd.get()); + return std::ptrdiff_t(reinterpret_cast(drvd.get()) + - reinterpret_cast(base0)); + }); + + py::classh(m, "Base0"); + py::classh(m, "Base1"); + py::classh(m, "Derived"); + + m.def( + "get_drvd_as_base0_raw_ptr", + []() { + auto *drvd = new Derived; + auto *base0 = dynamic_cast(drvd); + return base0; + }, + py::return_value_policy::take_ownership); + + m.def("get_drvd_as_base0_shared_ptr", []() { + auto drvd = std::make_shared(); + auto base0 = std::dynamic_pointer_cast(drvd); + return base0; + }); + + m.def("get_drvd_as_base0_unique_ptr", []() { + auto drvd = std::unique_ptr(new Derived); + auto base0 = std::unique_ptr(std::move(drvd)); + return base0; + }); + + m.def("vec_size_base0_raw_ptr", [](const Base0 *obj) { + const auto *obj_der = dynamic_cast(obj); + if (obj_der == nullptr) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); + + m.def("vec_size_base0_shared_ptr", [](const std::shared_ptr &obj) -> std::size_t { + const auto obj_der = std::dynamic_pointer_cast(obj); + if (!obj_der) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); + + m.def("vec_size_base0_unique_ptr", [](std::unique_ptr obj) -> std::size_t { + const auto *obj_der = dynamic_cast(obj.get()); + if (obj_der == nullptr) { + return std::size_t(0); + } + return obj_der->vec.size(); + }); +} diff --git a/tests/test_class_sh_mi_thunks.py b/tests/test_class_sh_mi_thunks.py new file mode 100644 index 0000000000..32bf47554b --- /dev/null +++ b/tests/test_class_sh_mi_thunks.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_mi_thunks as m + + +def test_ptrdiff_drvd_base0(): + ptrdiff = m.ptrdiff_drvd_base0() + # A failure here does not (necessarily) mean that there is a bug, but that + # test_class_sh_mi_thunks is not exercising what it is supposed to. + # If this ever fails on some platforms: use pytest.skip() + # If this ever fails on all platforms: don't know, seems extremely unlikely. + assert ptrdiff != 0 + + +@pytest.mark.parametrize( + "vec_size_fn", + [ + m.vec_size_base0_raw_ptr, + m.vec_size_base0_shared_ptr, + ], +) +@pytest.mark.parametrize( + "get_fn", + [ + m.get_drvd_as_base0_raw_ptr, + m.get_drvd_as_base0_shared_ptr, + m.get_drvd_as_base0_unique_ptr, + ], +) +def test_get_vec_size_raw_shared(get_fn, vec_size_fn): + obj = get_fn() + assert vec_size_fn(obj) == 5 + + +@pytest.mark.parametrize( + "get_fn", [m.get_drvd_as_base0_raw_ptr, m.get_drvd_as_base0_unique_ptr] +) +def test_get_vec_size_unique(get_fn): + obj = get_fn() + assert m.vec_size_base0_unique_ptr(obj) == 5 + with pytest.raises(ValueError, match="Python instance was disowned"): + m.vec_size_base0_unique_ptr(obj) + + +def test_get_shared_vec_size_unique(): + obj = m.get_drvd_as_base0_shared_ptr() + with pytest.raises(ValueError) as exc_info: + m.vec_size_base0_unique_ptr(obj) + assert ( + str(exc_info.value) == "Cannot disown external shared_ptr (load_as_unique_ptr)." + ) diff --git a/tests/test_class_sh_property.cpp b/tests/test_class_sh_property.cpp new file mode 100644 index 0000000000..6476a9cad8 --- /dev/null +++ b/tests/test_class_sh_property.cpp @@ -0,0 +1,95 @@ +// The compact 4-character naming matches that in test_class_sh_basic.cpp +// Variable names are intentionally terse, to not distract from the more important C++ type names: +// valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const, +// sh = shared_ptr, uq = unique_ptr. + +#include "pybind11/smart_holder.h" +#include "pybind11_tests.h" + +#include + +namespace test_class_sh_property { + +struct ClassicField { + int num = -88; +}; + +struct ClassicOuter { + ClassicField *m_mptr = nullptr; + const ClassicField *m_cptr = nullptr; +}; + +struct Field { + int num = -99; +}; + +struct Outer { + Field m_valu; + Field *m_mptr = nullptr; + const Field *m_cptr = nullptr; + std::unique_ptr m_uqmp; + std::unique_ptr m_uqcp; + std::shared_ptr m_shmp; + std::shared_ptr m_shcp; +}; + +inline void DisownOuter(std::unique_ptr) {} + +struct WithCharArrayMember { + WithCharArrayMember() { std::memcpy(char6_member, "Char6", 6); } + char char6_member[6]; +}; + +struct WithConstCharPtrMember { + const char *const_char_ptr_member = "ConstChar*"; +}; + +} // namespace test_class_sh_property + +TEST_SUBMODULE(class_sh_property, m) { + using namespace test_class_sh_property; + + py::class_>(m, "ClassicField") + .def(py::init<>()) + .def_readwrite("num", &ClassicField::num); + + py::class_>(m, "ClassicOuter") + .def(py::init<>()) + .def_readonly("m_mptr_readonly", &ClassicOuter::m_mptr) + .def_readwrite("m_mptr_readwrite", &ClassicOuter::m_mptr) + .def_readwrite("m_cptr_readonly", &ClassicOuter::m_cptr) + .def_readwrite("m_cptr_readwrite", &ClassicOuter::m_cptr); + + py::classh(m, "Field").def(py::init<>()).def_readwrite("num", &Field::num); + + py::classh(m, "Outer") + .def(py::init<>()) + + .def_readonly("m_valu_readonly", &Outer::m_valu) + .def_readwrite("m_valu_readwrite", &Outer::m_valu) + + .def_readonly("m_mptr_readonly", &Outer::m_mptr) + .def_readwrite("m_mptr_readwrite", &Outer::m_mptr) + .def_readonly("m_cptr_readonly", &Outer::m_cptr) + .def_readwrite("m_cptr_readwrite", &Outer::m_cptr) + + // .def_readonly("m_uqmp_readonly", &Outer::m_uqmp) // Custom compilation Error. + .def_readwrite("m_uqmp_readwrite", &Outer::m_uqmp) + // .def_readonly("m_uqcp_readonly", &Outer::m_uqcp) // Custom compilation Error. + .def_readwrite("m_uqcp_readwrite", &Outer::m_uqcp) + + .def_readwrite("m_shmp_readonly", &Outer::m_shmp) + .def_readwrite("m_shmp_readwrite", &Outer::m_shmp) + .def_readwrite("m_shcp_readonly", &Outer::m_shcp) + .def_readwrite("m_shcp_readwrite", &Outer::m_shcp); + + m.def("DisownOuter", DisownOuter); + + py::classh(m, "WithCharArrayMember") + .def(py::init<>()) + .def_readonly("char6_member", &WithCharArrayMember::char6_member); + + py::classh(m, "WithConstCharPtrMember") + .def(py::init<>()) + .def_readonly("const_char_ptr_member", &WithConstCharPtrMember::const_char_ptr_member); +} diff --git a/tests/test_class_sh_property.py b/tests/test_class_sh_property.py new file mode 100644 index 0000000000..0250a7f78e --- /dev/null +++ b/tests/test_class_sh_property.py @@ -0,0 +1,166 @@ +# The compact 4-character naming scheme (e.g. mptr, cptr, shcp) is explained at the top of +# test_class_sh_property.cpp. +from __future__ import annotations + +import pytest + +import env # noqa: F401 +from pybind11_tests import class_sh_property as m + + +@pytest.mark.skipif( + "env.PYPY or env.GRAALPY", reason="gc after `del field` is apparently deferred" +) +@pytest.mark.parametrize("m_attr", ["m_valu_readonly", "m_valu_readwrite"]) +def test_valu_getter(m_attr): + # Reduced from PyCLIF test: + # https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/testing/python/nested_fields_test.py#L56 + outer = m.Outer() + field = getattr(outer, m_attr) + assert field.num == -99 + with pytest.raises(ValueError) as excinfo: + m.DisownOuter(outer) + assert str(excinfo.value) == "Cannot disown use_count != 1 (load_as_unique_ptr)." + del field + m.DisownOuter(outer) + with pytest.raises(ValueError, match="Python instance was disowned") as excinfo: + getattr(outer, m_attr) + + +def test_valu_setter(): + outer = m.Outer() + assert outer.m_valu_readonly.num == -99 + assert outer.m_valu_readwrite.num == -99 + field = m.Field() + field.num = 35 + outer.m_valu_readwrite = field + assert outer.m_valu_readonly.num == 35 + assert outer.m_valu_readwrite.num == 35 + + +@pytest.mark.parametrize("m_attr", ["m_shmp", "m_shcp"]) +def test_shp(m_attr): + m_attr_readonly = m_attr + "_readonly" + m_attr_readwrite = m_attr + "_readwrite" + outer = m.Outer() + assert getattr(outer, m_attr_readonly) is None + assert getattr(outer, m_attr_readwrite) is None + field = m.Field() + field.num = 43 + setattr(outer, m_attr_readwrite, field) + assert getattr(outer, m_attr_readonly).num == 43 + assert getattr(outer, m_attr_readwrite).num == 43 + getattr(outer, m_attr_readonly).num = 57 + getattr(outer, m_attr_readwrite).num = 57 + assert field.num == 57 + del field + assert getattr(outer, m_attr_readonly).num == 57 + assert getattr(outer, m_attr_readwrite).num == 57 + + +@pytest.mark.parametrize( + ("field_type", "num_default", "outer_type"), + [ + (m.ClassicField, -88, m.ClassicOuter), + (m.Field, -99, m.Outer), + ], +) +@pytest.mark.parametrize("m_attr", ["m_mptr", "m_cptr"]) +@pytest.mark.parametrize("r_kind", ["_readonly", "_readwrite"]) +def test_ptr(field_type, num_default, outer_type, m_attr, r_kind): + m_attr_r_kind = m_attr + r_kind + outer = outer_type() + assert getattr(outer, m_attr_r_kind) is None + field = field_type() + assert field.num == num_default + setattr(outer, m_attr + "_readwrite", field) + assert getattr(outer, m_attr_r_kind).num == num_default + field.num = 76 + assert getattr(outer, m_attr_r_kind).num == 76 + # Change to -88 or -99 to demonstrate Undefined Behavior (dangling pointer). + if num_default == 88 and m_attr == "m_mptr": + del field + assert getattr(outer, m_attr_r_kind).num == 76 + + +@pytest.mark.parametrize("m_attr_readwrite", ["m_uqmp_readwrite", "m_uqcp_readwrite"]) +def test_uqp(m_attr_readwrite): + outer = m.Outer() + assert getattr(outer, m_attr_readwrite) is None + field_orig = m.Field() + field_orig.num = 39 + setattr(outer, m_attr_readwrite, field_orig) + with pytest.raises(ValueError, match="Python instance was disowned"): + _ = field_orig.num + field_retr1 = getattr(outer, m_attr_readwrite) + assert getattr(outer, m_attr_readwrite) is None + assert field_retr1.num == 39 + field_retr1.num = 93 + setattr(outer, m_attr_readwrite, field_retr1) + with pytest.raises(ValueError): + _ = field_retr1.num + field_retr2 = getattr(outer, m_attr_readwrite) + assert field_retr2.num == 93 + + +# Proof-of-concept (POC) for safe & intuitive Python access to unique_ptr members. +# The C++ member unique_ptr is disowned to a temporary Python object for accessing +# an attribute of the member. After the attribute was accessed, the Python object +# is disowned back to the C++ member unique_ptr. +# Productizing this POC is left for a future separate PR, as needed. +class unique_ptr_field_proxy_poc: + def __init__(self, obj, field_name): + object.__setattr__(self, "__obj", obj) + object.__setattr__(self, "__field_name", field_name) + + def __getattr__(self, *args, **kwargs): + return _proxy_dereference(self, getattr, *args, **kwargs) + + def __setattr__(self, *args, **kwargs): + return _proxy_dereference(self, setattr, *args, **kwargs) + + def __delattr__(self, *args, **kwargs): + return _proxy_dereference(self, delattr, *args, **kwargs) + + +def _proxy_dereference(proxy, xxxattr, *args, **kwargs): + obj = object.__getattribute__(proxy, "__obj") + field_name = object.__getattribute__(proxy, "__field_name") + field = getattr(obj, field_name) # Disowns the C++ unique_ptr member. + assert field is not None + try: + return xxxattr(field, *args, **kwargs) + finally: + setattr(obj, field_name, field) # Disowns the temporary Python object (field). + + +@pytest.mark.parametrize("m_attr", ["m_uqmp", "m_uqcp"]) +def test_unique_ptr_field_proxy_poc(m_attr): + m_attr_readwrite = m_attr + "_readwrite" + outer = m.Outer() + field_orig = m.Field() + field_orig.num = 45 + setattr(outer, m_attr_readwrite, field_orig) + field_proxy = unique_ptr_field_proxy_poc(outer, m_attr_readwrite) + assert field_proxy.num == 45 + assert field_proxy.num == 45 + with pytest.raises(AttributeError): + _ = field_proxy.xyz + assert field_proxy.num == 45 + field_proxy.num = 82 + assert field_proxy.num == 82 + field_proxy = unique_ptr_field_proxy_poc(outer, m_attr_readwrite) + assert field_proxy.num == 82 + with pytest.raises(AttributeError): + del field_proxy.num + assert field_proxy.num == 82 + + +def test_readonly_char6_member(): + obj = m.WithCharArrayMember() + assert obj.char6_member == "Char6" + + +def test_readonly_const_char_ptr_member(): + obj = m.WithConstCharPtrMember() + assert obj.const_char_ptr_member == "ConstChar*" diff --git a/tests/test_class_sh_property_non_owning.cpp b/tests/test_class_sh_property_non_owning.cpp new file mode 100644 index 0000000000..42f1f6fd35 --- /dev/null +++ b/tests/test_class_sh_property_non_owning.cpp @@ -0,0 +1,64 @@ +#include "pybind11/smart_holder.h" +#include "pybind11_tests.h" + +#include +#include + +namespace test_class_sh_property_non_owning { + +struct CoreField { + explicit CoreField(int int_value = -99) : int_value{int_value} {} + int int_value; +}; + +struct DataField { + DataField(int i_value, int i_shared, int i_unique) + : core_fld_value{i_value}, core_fld_shared_ptr{new CoreField{i_shared}}, + core_fld_raw_ptr{core_fld_shared_ptr.get()}, + core_fld_unique_ptr{new CoreField{i_unique}} {} + CoreField core_fld_value; + std::shared_ptr core_fld_shared_ptr; + CoreField *core_fld_raw_ptr; + std::unique_ptr core_fld_unique_ptr; +}; + +struct DataFieldsHolder { +private: + std::vector vec; + +public: + explicit DataFieldsHolder(std::size_t vec_size) { + for (std::size_t i = 0; i < vec_size; i++) { + int i11 = static_cast(i) * 11; + vec.emplace_back(13 + i11, 14 + i11, 15 + i11); + } + } + + DataField *vec_at(std::size_t index) { + if (index >= vec.size()) { + return nullptr; + } + return &vec[index]; + } +}; + +} // namespace test_class_sh_property_non_owning + +using namespace test_class_sh_property_non_owning; + +TEST_SUBMODULE(class_sh_property_non_owning, m) { + py::classh(m, "CoreField").def_readwrite("int_value", &CoreField::int_value); + + py::classh(m, "DataField") + .def_readonly("core_fld_value_ro", &DataField::core_fld_value) + .def_readwrite("core_fld_value_rw", &DataField::core_fld_value) + .def_readonly("core_fld_shared_ptr_ro", &DataField::core_fld_shared_ptr) + .def_readwrite("core_fld_shared_ptr_rw", &DataField::core_fld_shared_ptr) + .def_readonly("core_fld_raw_ptr_ro", &DataField::core_fld_raw_ptr) + .def_readwrite("core_fld_raw_ptr_rw", &DataField::core_fld_raw_ptr) + .def_readwrite("core_fld_unique_ptr_rw", &DataField::core_fld_unique_ptr); + + py::classh(m, "DataFieldsHolder") + .def(py::init()) + .def("vec_at", &DataFieldsHolder::vec_at, py::return_value_policy::reference_internal); +} diff --git a/tests/test_class_sh_property_non_owning.py b/tests/test_class_sh_property_non_owning.py new file mode 100644 index 0000000000..33a9d4503b --- /dev/null +++ b/tests/test_class_sh_property_non_owning.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_property_non_owning as m + + +@pytest.mark.parametrize("persistent_holder", [True, False]) +@pytest.mark.parametrize( + ("core_fld", "expected"), + [ + ("core_fld_value_ro", (13, 24)), + ("core_fld_value_rw", (13, 24)), + ("core_fld_shared_ptr_ro", (14, 25)), + ("core_fld_shared_ptr_rw", (14, 25)), + ("core_fld_raw_ptr_ro", (14, 25)), + ("core_fld_raw_ptr_rw", (14, 25)), + ("core_fld_unique_ptr_rw", (15, 26)), + ], +) +def test_core_fld_common(core_fld, expected, persistent_holder): + if persistent_holder: + h = m.DataFieldsHolder(2) + for i, exp in enumerate(expected): + c = getattr(h.vec_at(i), core_fld) + assert c.int_value == exp + else: + for i, exp in enumerate(expected): + c = getattr(m.DataFieldsHolder(2).vec_at(i), core_fld) + assert c.int_value == exp diff --git a/tests/test_class_sh_shared_ptr_copy_move.cpp b/tests/test_class_sh_shared_ptr_copy_move.cpp new file mode 100644 index 0000000000..5eb0ef3edb --- /dev/null +++ b/tests/test_class_sh_shared_ptr_copy_move.cpp @@ -0,0 +1,105 @@ +#include + +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace { + +const std::string fooNames[] = {"ShPtr_", "SmHld_"}; + +template +struct Foo { + std::string history; + explicit Foo(const std::string &history_) : history(history_) {} + Foo(const Foo &other) : history(other.history + "_CpCtor") {} + Foo(Foo &&other) noexcept : history(other.history + "_MvCtor") {} + Foo &operator=(const Foo &other) { + history = other.history + "_OpEqLv"; + return *this; + } + Foo &operator=(Foo &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + std::string get_history() const { return "Foo" + fooNames[SerNo] + history; } +}; + +using FooShPtr = Foo<0>; +using FooSmHld = Foo<1>; + +struct Outer { + std::shared_ptr ShPtr; + std::shared_ptr SmHld; + Outer() + : ShPtr(std::make_shared("Outer")), SmHld(std::make_shared("Outer")) {} + std::shared_ptr getShPtr() const { return ShPtr; } + std::shared_ptr getSmHld() const { return SmHld; } +}; + +} // namespace + +TEST_SUBMODULE(class_sh_shared_ptr_copy_move, m) { + namespace py = pybind11; + + py::class_>(m, "FooShPtr") + .def("get_history", &FooShPtr::get_history); + py::classh(m, "FooSmHld").def("get_history", &FooSmHld::get_history); + + auto outer = py::class_(m, "Outer").def(py::init()); +#define MAKE_PROP(PropTyp) \ + MAKE_PROP_FOO(ShPtr, PropTyp) \ + MAKE_PROP_FOO(SmHld, PropTyp) + +#define MAKE_PROP_FOO(FooTyp, PropTyp) \ + .def_##PropTyp(#FooTyp "_" #PropTyp "_default", &Outer::FooTyp) \ + .def_##PropTyp( \ + #FooTyp "_" #PropTyp "_copy", &Outer::FooTyp, py::return_value_policy::copy) \ + .def_##PropTyp( \ + #FooTyp "_" #PropTyp "_move", &Outer::FooTyp, py::return_value_policy::move) + outer MAKE_PROP(readonly) MAKE_PROP(readwrite); +#undef MAKE_PROP_FOO + +#define MAKE_PROP_FOO(FooTyp, PropTyp) \ + .def_##PropTyp(#FooTyp "_property_" #PropTyp "_default", &Outer::FooTyp) \ + .def_property_##PropTyp(#FooTyp "_property_" #PropTyp "_copy", \ + &Outer::get##FooTyp, \ + py::return_value_policy::copy) \ + .def_property_##PropTyp(#FooTyp "_property_" #PropTyp "_move", \ + &Outer::get##FooTyp, \ + py::return_value_policy::move) + outer MAKE_PROP(readonly); +#undef MAKE_PROP_FOO +#undef MAKE_PROP + + m.def("test_ShPtr_copy", []() { + auto o = std::make_shared("copy"); + auto l = py::list(); + l.append(o); + return l; + }); + m.def("test_SmHld_copy", []() { + auto o = std::make_shared("copy"); + auto l = py::list(); + l.append(o); + return l; + }); + + m.def("test_ShPtr_move", []() { + auto o = std::make_shared("move"); + auto l = py::list(); + l.append(std::move(o)); + return l; + }); + m.def("test_SmHld_move", []() { + auto o = std::make_shared("move"); + auto l = py::list(); + l.append(std::move(o)); + return l; + }); +} + +} // namespace pybind11_tests diff --git a/tests/test_class_sh_shared_ptr_copy_move.py b/tests/test_class_sh_shared_ptr_copy_move.py new file mode 100644 index 0000000000..067bb47d2a --- /dev/null +++ b/tests/test_class_sh_shared_ptr_copy_move.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_shared_ptr_copy_move as m + + +def test_shptr_copy(): + txt = m.test_ShPtr_copy()[0].get_history() + assert txt == "FooShPtr_copy" + + +def test_smhld_copy(): + txt = m.test_SmHld_copy()[0].get_history() + assert txt == "FooSmHld_copy" + + +def test_shptr_move(): + txt = m.test_ShPtr_move()[0].get_history() + assert txt == "FooShPtr_move" + + +def test_smhld_move(): + txt = m.test_SmHld_move()[0].get_history() + assert txt == "FooSmHld_move" + + +def _check_property(foo_typ, prop_typ, policy): + o = m.Outer() + name = f"{foo_typ}_{prop_typ}_{policy}" + history = f"Foo{foo_typ}_Outer" + f = getattr(o, name) + assert f.get_history() == history + # and try again to check that o did not get changed + f = getattr(o, name) + assert f.get_history() == history + + +def test_properties(): + for prop_typ in ("readonly", "readwrite", "property_readonly"): + for foo_typ in ("ShPtr", "SmHld"): + for policy in ("default", "copy", "move"): + _check_property(foo_typ, prop_typ, policy) diff --git a/tests/test_class_sh_trampoline_basic.cpp b/tests/test_class_sh_trampoline_basic.cpp new file mode 100644 index 0000000000..7ae3be5bf9 --- /dev/null +++ b/tests/test_class_sh_trampoline_basic.cpp @@ -0,0 +1,84 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_basic { + +template // Using int as a trick to easily generate a series of types. +struct Abase { + int val = 0; + virtual ~Abase() = default; + explicit Abase(int val_) : val{val_} {} + int Get() const { return val * 10 + 3; } + virtual int Add(int other_val) const = 0; + + // Some compilers complain about implicitly defined versions of some of the following: + Abase(const Abase &) = default; + Abase(Abase &&) noexcept = default; + Abase &operator=(const Abase &) = default; + Abase &operator=(Abase &&) noexcept = default; +}; + +template +struct AbaseAlias : Abase { + using Abase::Abase; + + int Add(int other_val) const override { + PYBIND11_OVERRIDE_PURE(int, /* Return type */ + Abase, /* Parent class */ + Add, /* Name of function in C++ (must match Python name) */ + other_val); + } +}; + +template <> +struct AbaseAlias<1> : Abase<1>, py::trampoline_self_life_support { + using Abase<1>::Abase; + + int Add(int other_val) const override { + PYBIND11_OVERRIDE_PURE(int, /* Return type */ + Abase<1>, /* Parent class */ + Add, /* Name of function in C++ (must match Python name) */ + other_val); + } +}; + +template +int AddInCppRawPtr(const Abase *obj, int other_val) { + return obj->Add(other_val) * 10 + 7; +} + +template +int AddInCppSharedPtr(std::shared_ptr> obj, int other_val) { + return obj->Add(other_val) * 100 + 11; +} + +template +int AddInCppUniquePtr(std::unique_ptr> obj, int other_val) { + return obj->Add(other_val) * 100 + 13; +} + +template +void wrap(py::module_ m, const char *py_class_name) { + py::classh, AbaseAlias>(m, py_class_name) + .def(py::init(), py::arg("val")) + .def("Get", &Abase::Get) + .def("Add", &Abase::Add, py::arg("other_val")); + + m.def("AddInCppRawPtr", AddInCppRawPtr, py::arg("obj"), py::arg("other_val")); + m.def("AddInCppSharedPtr", AddInCppSharedPtr, py::arg("obj"), py::arg("other_val")); + m.def("AddInCppUniquePtr", AddInCppUniquePtr, py::arg("obj"), py::arg("other_val")); +} + +} // namespace class_sh_trampoline_basic +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_basic; + +TEST_SUBMODULE(class_sh_trampoline_basic, m) { + wrap<0>(m, "Abase0"); + wrap<1>(m, "Abase1"); +} diff --git a/tests/test_class_sh_trampoline_basic.py b/tests/test_class_sh_trampoline_basic.py new file mode 100644 index 0000000000..eab82121fb --- /dev/null +++ b/tests/test_class_sh_trampoline_basic.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_trampoline_basic as m + + +class PyDrvd0(m.Abase0): + def __init__(self, val): + super().__init__(val) + + def Add(self, other_val): + return self.Get() * 100 + other_val + + +class PyDrvd1(m.Abase1): + def __init__(self, val): + super().__init__(val) + + def Add(self, other_val): + return self.Get() * 200 + other_val + + +def test_drvd0_add(): + drvd = PyDrvd0(74) + assert drvd.Add(38) == (74 * 10 + 3) * 100 + 38 + + +def test_drvd0_add_in_cpp_raw_ptr(): + drvd = PyDrvd0(52) + assert m.AddInCppRawPtr(drvd, 27) == ((52 * 10 + 3) * 100 + 27) * 10 + 7 + + +def test_drvd0_add_in_cpp_shared_ptr(): + while True: + drvd = PyDrvd0(36) + assert m.AddInCppSharedPtr(drvd, 56) == ((36 * 10 + 3) * 100 + 56) * 100 + 11 + return # Comment out for manual leak checking (use `top` command). + + +def test_drvd0_add_in_cpp_unique_ptr(): + while True: + drvd = PyDrvd0(0) + with pytest.raises(ValueError) as exc_info: + m.AddInCppUniquePtr(drvd, 0) + assert ( + str(exc_info.value) + == "Alias class (also known as trampoline) does not inherit from" + " py::trampoline_self_life_support, therefore the ownership of this" + " instance cannot safely be transferred to C++." + ) + return # Comment out for manual leak checking (use `top` command). + + +def test_drvd1_add_in_cpp_unique_ptr(): + while True: + drvd = PyDrvd1(25) + assert m.AddInCppUniquePtr(drvd, 83) == ((25 * 10 + 3) * 200 + 83) * 100 + 13 + return # Comment out for manual leak checking (use `top` command). diff --git a/tests/test_class_sh_trampoline_self_life_support.cpp b/tests/test_class_sh_trampoline_self_life_support.cpp new file mode 100644 index 0000000000..6c14e39c99 --- /dev/null +++ b/tests/test_class_sh_trampoline_self_life_support.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/smart_holder.h" +#include "pybind11/trampoline_self_life_support.h" +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_self_life_support { + +struct Big5 { // Also known as "rule of five". + std::string history; + + explicit Big5(std::string history_start) : history{std::move(history_start)} {} + + Big5(const Big5 &other) { history = other.history + "_CpCtor"; } + + Big5(Big5 &&other) noexcept { history = other.history + "_MvCtor"; } + + Big5 &operator=(const Big5 &other) { + history = other.history + "_OpEqLv"; + return *this; + } + + Big5 &operator=(Big5 &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + + virtual ~Big5() = default; + +protected: + Big5() : history{"DefaultConstructor"} {} +}; + +struct Big5Trampoline : Big5, py::trampoline_self_life_support { + using Big5::Big5; +}; + +} // namespace class_sh_trampoline_self_life_support +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_self_life_support; + +TEST_SUBMODULE(class_sh_trampoline_self_life_support, m) { + py::classh(m, "Big5") + .def(py::init()) + .def_readonly("history", &Big5::history); + + m.def("action", [](std::unique_ptr obj, int action_id) { + py::object o2 = py::none(); + // This is very unusual, but needed to directly exercise the trampoline_self_life_support + // CpCtor, MvCtor, operator= lvalue, operator= rvalue. + auto *obj_trampoline = dynamic_cast(obj.get()); + if (obj_trampoline != nullptr) { + switch (action_id) { + case 0: { // CpCtor + std::unique_ptr cp(new Big5Trampoline(*obj_trampoline)); + o2 = py::cast(std::move(cp)); + } break; + case 1: { // MvCtor + std::unique_ptr mv(new Big5Trampoline(std::move(*obj_trampoline))); + o2 = py::cast(std::move(mv)); + } break; + case 2: { // operator= lvalue + std::unique_ptr lv(new Big5Trampoline); + *lv = *obj_trampoline; // NOLINT clang-tidy cppcoreguidelines-slicing + o2 = py::cast(std::move(lv)); + } break; + case 3: { // operator= rvalue + std::unique_ptr rv(new Big5Trampoline); + *rv = std::move(*obj_trampoline); + o2 = py::cast(std::move(rv)); + } break; + default: + break; + } + } + py::object o1 = py::cast(std::move(obj)); + return py::make_tuple(o1, o2); + }); +} diff --git a/tests/test_class_sh_trampoline_self_life_support.py b/tests/test_class_sh_trampoline_self_life_support.py new file mode 100644 index 0000000000..d4af2ab99a --- /dev/null +++ b/tests/test_class_sh_trampoline_self_life_support.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest + +import pybind11_tests.class_sh_trampoline_self_life_support as m + + +class PyBig5(m.Big5): + pass + + +def test_m_big5(): + obj = m.Big5("Seed") + assert obj.history == "Seed" + o1, o2 = m.action(obj, 0) + assert o1 is not obj + assert o1.history == "Seed" + with pytest.raises(ValueError) as excinfo: + _ = obj.history + assert "Python instance was disowned" in str(excinfo.value) + assert o2 is None + + +@pytest.mark.parametrize( + ("action_id", "expected_history"), + [ + (0, "Seed_CpCtor"), + (1, "Seed_MvCtor"), + (2, "Seed_OpEqLv"), + (3, "Seed_OpEqRv"), + ], +) +def test_py_big5(action_id, expected_history): + obj = PyBig5("Seed") + assert obj.history == "Seed" + o1, o2 = m.action(obj, action_id) + assert o1 is obj + assert o2.history == expected_history diff --git a/tests/test_class_sh_trampoline_shared_from_this.cpp b/tests/test_class_sh_trampoline_shared_from_this.cpp new file mode 100644 index 0000000000..9ccc213298 --- /dev/null +++ b/tests/test_class_sh_trampoline_shared_from_this.cpp @@ -0,0 +1,138 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/smart_holder.h" +#include "pybind11_tests.h" + +#include +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_shared_from_this { + +struct Sft : std::enable_shared_from_this { + std::string history; + explicit Sft(const std::string &history_seed) : history{history_seed} {} + virtual ~Sft() = default; + +#if defined(__clang__) + // "Group of 4" begin. + // This group is not meant to be used, but will leave a trace in the + // history in case something goes wrong. + // However, compilers other than clang have a variety of issues. It is not + // worth the trouble covering all platforms. + Sft(const Sft &other) : enable_shared_from_this(other) { history = other.history + "_CpCtor"; } + + Sft(Sft &&other) noexcept { history = other.history + "_MvCtor"; } + + Sft &operator=(const Sft &other) { + history = other.history + "_OpEqLv"; + return *this; + } + + Sft &operator=(Sft &&other) noexcept { + history = other.history + "_OpEqRv"; + return *this; + } + // "Group of 4" end. +#endif +}; + +struct SftSharedPtrStash { + int ser_no; + std::vector> stash; + explicit SftSharedPtrStash(int ser_no) : ser_no{ser_no} {} + void Clear() { stash.clear(); } + void Add(const std::shared_ptr &obj) { + if (!obj->history.empty()) { + obj->history += "_Stash" + std::to_string(ser_no) + "Add"; + } + stash.push_back(obj); + } + void AddSharedFromThis(Sft *obj) { + auto sft = obj->shared_from_this(); + if (!sft->history.empty()) { + sft->history += "_Stash" + std::to_string(ser_no) + "AddSharedFromThis"; + } + stash.push_back(sft); + } + std::string history(unsigned i) { + if (i < stash.size()) { + return stash[i]->history; + } + return "OutOfRange"; + } + long use_count(unsigned i) { + if (i < stash.size()) { + return stash[i].use_count(); + } + return -1; + } +}; + +struct SftTrampoline : Sft, py::trampoline_self_life_support { + using Sft::Sft; +}; + +long use_count(const std::shared_ptr &obj) { return obj.use_count(); } + +long pass_shared_ptr(const std::shared_ptr &obj) { + auto sft = obj->shared_from_this(); + if (!sft->history.empty()) { + sft->history += "_PassSharedPtr"; + } + return sft.use_count(); +} + +std::string pass_unique_ptr_cref(const std::unique_ptr &obj) { + return obj ? obj->history : ""; +} +void pass_unique_ptr_rref(std::unique_ptr &&) { + throw std::runtime_error("Expected to not be reached."); +} + +Sft *make_pure_cpp_sft_raw_ptr(const std::string &history_seed) { return new Sft{history_seed}; } + +std::unique_ptr make_pure_cpp_sft_unq_ptr(const std::string &history_seed) { + return std::unique_ptr(new Sft{history_seed}); +} + +std::shared_ptr make_pure_cpp_sft_shd_ptr(const std::string &history_seed) { + return std::make_shared(history_seed); +} + +std::shared_ptr pass_through_shd_ptr(const std::shared_ptr &obj) { return obj; } + +} // namespace class_sh_trampoline_shared_from_this +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_shared_from_this; + +TEST_SUBMODULE(class_sh_trampoline_shared_from_this, m) { + py::classh(m, "Sft") + .def(py::init()) + .def(py::init([](const std::string &history, int) { + return std::make_shared(history); + })) + .def_readonly("history", &Sft::history) + // This leads to multiple entries in registered_instances: + .def(py::init([](const std::shared_ptr &existing) { return existing; })); + + py::classh(m, "SftSharedPtrStash") + .def(py::init()) + .def("Clear", &SftSharedPtrStash::Clear) + .def("Add", &SftSharedPtrStash::Add) + .def("AddSharedFromThis", &SftSharedPtrStash::AddSharedFromThis) + .def("history", &SftSharedPtrStash::history) + .def("use_count", &SftSharedPtrStash::use_count); + + m.def("use_count", use_count); + m.def("pass_shared_ptr", pass_shared_ptr); + m.def("pass_unique_ptr_cref", pass_unique_ptr_cref); + m.def("pass_unique_ptr_rref", pass_unique_ptr_rref); + m.def("make_pure_cpp_sft_raw_ptr", make_pure_cpp_sft_raw_ptr); + m.def("make_pure_cpp_sft_unq_ptr", make_pure_cpp_sft_unq_ptr); + m.def("make_pure_cpp_sft_shd_ptr", make_pure_cpp_sft_shd_ptr); + m.def("pass_through_shd_ptr", pass_through_shd_ptr); +} diff --git a/tests/test_class_sh_trampoline_shared_from_this.py b/tests/test_class_sh_trampoline_shared_from_this.py new file mode 100644 index 0000000000..fbe31387a8 --- /dev/null +++ b/tests/test_class_sh_trampoline_shared_from_this.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import sys +import weakref + +import pytest + +import env +import pybind11_tests.class_sh_trampoline_shared_from_this as m + + +class PySft(m.Sft): + pass + + +def test_release_and_shared_from_this(): + # Exercises the most direct path from building a shared_from_this-visible + # shared_ptr to calling shared_from_this. + obj = PySft("PySft") + assert obj.history == "PySft" + assert m.use_count(obj) == 1 + assert m.pass_shared_ptr(obj) == 2 + assert obj.history == "PySft_PassSharedPtr" + assert m.use_count(obj) == 1 + assert m.pass_shared_ptr(obj) == 2 + assert obj.history == "PySft_PassSharedPtr_PassSharedPtr" + assert m.use_count(obj) == 1 + + +def test_release_and_shared_from_this_leak(): + obj = PySft("") + while True: + m.pass_shared_ptr(obj) + assert not obj.history + assert m.use_count(obj) == 1 + break # Comment out for manual leak checking (use `top` command). + + +def test_release_and_stash(): + # Exercises correct functioning of guarded_delete weak_ptr. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) + exp_hist = "PySft_Stash1Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 2 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + assert m.pass_shared_ptr(obj) == 3 + exp_hist += "_PassSharedPtr" + assert obj.history == exp_hist + assert m.use_count(obj) == 2 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + stash2 = m.SftSharedPtrStash(2) + stash2.Add(obj) + exp_hist += "_Stash2Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 3 + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 2 + stash2.Add(obj) + exp_hist += "_Stash2Add" + assert obj.history == exp_hist + assert m.use_count(obj) == 4 + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 3 + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 3 + assert stash2.history(1) == exp_hist + assert stash2.use_count(1) == 3 + del obj + assert stash2.history(0) == exp_hist + assert stash2.use_count(0) == 3 + assert stash2.history(1) == exp_hist + assert stash2.use_count(1) == 3 + stash2.Clear() + assert stash1.history(0) == exp_hist + assert stash1.use_count(0) == 1 + + +def test_release_and_stash_leak(): + obj = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) + assert not obj.history + assert m.use_count(obj) == 2 + assert stash1.use_count(0) == 1 + stash1.Add(obj) + assert not obj.history + assert m.use_count(obj) == 3 + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + break # Comment out for manual leak checking (use `top` command). + + +def test_release_and_stash_via_shared_from_this(): + # Exercises that the smart_holder vptr is invisible to the shared_from_this mechanism. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + with pytest.raises(RuntimeError) as exc_info: + stash1.AddSharedFromThis(obj) + assert str(exc_info.value) == "bad_weak_ptr" + stash1.Add(obj) + assert obj.history == "PySft_Stash1Add" + assert stash1.use_count(0) == 1 + stash1.AddSharedFromThis(obj) + assert obj.history == "PySft_Stash1Add_Stash1AddSharedFromThis" + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + + +def test_release_and_stash_via_shared_from_this_leak(): + obj = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + with pytest.raises(RuntimeError) as exc_info: + stash1.AddSharedFromThis(obj) + assert str(exc_info.value) == "bad_weak_ptr" + stash1.Add(obj) + assert not obj.history + assert stash1.use_count(0) == 1 + stash1.AddSharedFromThis(obj) + assert not obj.history + assert stash1.use_count(0) == 2 + assert stash1.use_count(1) == 2 + break # Comment out for manual leak checking (use `top` command). + + +def test_pass_released_shared_ptr_as_unique_ptr(): + # Exercises that returning a unique_ptr fails while a shared_from_this + # visible shared_ptr exists. + obj = PySft("PySft") + stash1 = m.SftSharedPtrStash(1) + stash1.Add(obj) # Releases shared_ptr to C++. + assert m.pass_unique_ptr_cref(obj) == "PySft_Stash1Add" + assert obj.history == "PySft_Stash1Add" + with pytest.raises(ValueError) as exc_info: + m.pass_unique_ptr_rref(obj) + assert str(exc_info.value) == ( + "Python instance is currently owned by a std::shared_ptr." + ) + assert obj.history == "PySft_Stash1Add" + + +@pytest.mark.parametrize( + "make_f", + [ + m.make_pure_cpp_sft_raw_ptr, + m.make_pure_cpp_sft_unq_ptr, + m.make_pure_cpp_sft_shd_ptr, + ], +) +def test_pure_cpp_sft_raw_ptr(make_f): + # Exercises void_cast_raw_ptr logic for different situations. + obj = make_f("PureCppSft") + assert m.pass_shared_ptr(obj) == 3 + assert obj.history == "PureCppSft_PassSharedPtr" + obj = make_f("PureCppSft") + stash1 = m.SftSharedPtrStash(1) + stash1.AddSharedFromThis(obj) + assert obj.history == "PureCppSft_Stash1AddSharedFromThis" + + +def test_multiple_registered_instances_for_same_pointee(): + obj0 = PySft("PySft") + obj0.attachment_in_dict = "Obj0" + assert m.pass_through_shd_ptr(obj0) is obj0 + while True: + obj = m.Sft(obj0) + assert obj is not obj0 + obj_pt = m.pass_through_shd_ptr(obj) + # Unpredictable! Because registered_instances is as std::unordered_multimap. + assert obj_pt is obj0 or obj_pt is obj + # Multiple registered_instances for the same pointee can lead to unpredictable results: + if obj_pt is obj0: + assert obj_pt.attachment_in_dict == "Obj0" + else: + assert not hasattr(obj_pt, "attachment_in_dict") + assert obj0.history == "PySft" + break # Comment out for manual leak checking (use `top` command). + + +def test_multiple_registered_instances_for_same_pointee_leak(): + obj0 = PySft("") + while True: + stash1 = m.SftSharedPtrStash(1) + stash1.Add(m.Sft(obj0)) + assert stash1.use_count(0) == 1 + stash1.Add(m.Sft(obj0)) + assert stash1.use_count(0) == 1 + assert stash1.use_count(1) == 1 + assert not obj0.history + break # Comment out for manual leak checking (use `top` command). + + +def test_multiple_registered_instances_for_same_pointee_recursive(): + while True: + obj0 = PySft("PySft") + if not env.PYPY: + obj0_wr = weakref.ref(obj0) + obj = obj0 + # This loop creates a chain of instances linked by shared_ptrs. + for _ in range(10): + obj_next = m.Sft(obj) + assert obj_next is not obj + obj = obj_next + del obj_next + assert obj.history == "PySft" + del obj0 + if not env.PYPY and not env.GRAALPY: + assert obj0_wr() is not None + del obj # This releases the chain recursively. + if not env.PYPY and not env.GRAALPY: + assert obj0_wr() is None + break # Comment out for manual leak checking (use `top` command). + + +# As of 2021-07-10 the pybind11 GitHub Actions valgrind build uses Python 3.9. +WORKAROUND_ENABLING_ROLLBACK_OF_PR3068 = env.LINUX and sys.version_info == (3, 9) + + +def test_std_make_shared_factory(): + class PySftMakeShared(m.Sft): + def __init__(self, history): + super().__init__(history, 0) + + obj = PySftMakeShared("PySftMakeShared") + assert obj.history == "PySftMakeShared" + if WORKAROUND_ENABLING_ROLLBACK_OF_PR3068: + try: + m.pass_through_shd_ptr(obj) + except RuntimeError as e: + str_exc_info_value = str(e) + else: + str_exc_info_value = "RuntimeError NOT RAISED" + else: + with pytest.raises(RuntimeError) as exc_info: + m.pass_through_shd_ptr(obj) + str_exc_info_value = str(exc_info.value) + assert ( + str_exc_info_value + == "smart_holder_type_casters load_as_shared_ptr failure: not implemented:" + " trampoline-self-life-support for external shared_ptr to type inheriting" + " from std::enable_shared_from_this." + ) diff --git a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp new file mode 100644 index 0000000000..ee381f2eb8 --- /dev/null +++ b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/smart_holder.h" +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_shared_ptr_cpp_arg { + +// For testing whether a python subclass of a C++ object dies when the +// last python reference is lost +struct SpBase { + // returns true if the base virtual function is called + virtual bool is_base_used() { return true; } + + // returns true if there's an associated python instance + bool has_python_instance() { + auto *tinfo = py::detail::get_type_info(typeid(SpBase)); + return (bool) py::detail::get_object_handle(this, tinfo); + } + + SpBase() = default; + SpBase(const SpBase &) = delete; + virtual ~SpBase() = default; +}; + +std::shared_ptr pass_through_shd_ptr(const std::shared_ptr &obj) { return obj; } + +struct PySpBase : SpBase { + using SpBase::SpBase; + bool is_base_used() override { PYBIND11_OVERRIDE(bool, SpBase, is_base_used); } +}; + +struct SpBaseTester { + std::shared_ptr get_object() const { return m_obj; } + void set_object(std::shared_ptr obj) { m_obj = std::move(obj); } + bool is_base_used() { return m_obj->is_base_used(); } + bool has_instance() { return (bool) m_obj; } + bool has_python_instance() { return m_obj && m_obj->has_python_instance(); } + void set_nonpython_instance() { m_obj = std::make_shared(); } + std::shared_ptr m_obj; +}; + +// For testing that a C++ class without an alias does not retain the python +// portion of the object +struct SpGoAway {}; + +struct SpGoAwayTester { + std::shared_ptr m_obj; +}; + +} // namespace class_sh_trampoline_shared_ptr_cpp_arg +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_trampoline_shared_ptr_cpp_arg; + +TEST_SUBMODULE(class_sh_trampoline_shared_ptr_cpp_arg, m) { + // For testing whether a python subclass of a C++ object dies when the + // last python reference is lost + + py::classh(m, "SpBase") + .def(py::init<>()) + .def(py::init([](int) { return std::make_shared(); })) + .def("is_base_used", &SpBase::is_base_used) + .def("has_python_instance", &SpBase::has_python_instance); + + m.def("pass_through_shd_ptr", pass_through_shd_ptr); + m.def("pass_through_shd_ptr_release_gil", + pass_through_shd_ptr, + py::call_guard()); // PR #4196 + + py::classh(m, "SpBaseTester") + .def(py::init<>()) + .def("get_object", &SpBaseTester::get_object) + .def("set_object", &SpBaseTester::set_object) + .def("is_base_used", &SpBaseTester::is_base_used) + .def("has_instance", &SpBaseTester::has_instance) + .def("has_python_instance", &SpBaseTester::has_python_instance) + .def("set_nonpython_instance", &SpBaseTester::set_nonpython_instance) + .def_readwrite("obj", &SpBaseTester::m_obj); + + // For testing that a C++ class without an alias does not retain the python + // portion of the object + + py::classh(m, "SpGoAway").def(py::init<>()); + + py::classh(m, "SpGoAwayTester") + .def(py::init<>()) + .def_readwrite("obj", &SpGoAwayTester::m_obj); +} diff --git a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py new file mode 100644 index 0000000000..a693621e3b --- /dev/null +++ b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import pytest + +import env # noqa: F401 +import pybind11_tests.class_sh_trampoline_shared_ptr_cpp_arg as m + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_cpp_arg(): + import weakref + + class PyChild(m.SpBase): + def is_base_used(self): + return False + + tester = m.SpBaseTester() + + obj = PyChild() + objref = weakref.ref(obj) + + # Pass the last python reference to the C++ function + tester.set_object(obj) + del obj + pytest.gc_collect() + + # python reference is still around since C++ has it now + assert objref() is not None + assert tester.is_base_used() is False + assert tester.obj.is_base_used() is False + assert tester.get_object() is objref() + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_cpp_prop(): + class PyChild(m.SpBase): + def is_base_used(self): + return False + + tester = m.SpBaseTester() + + # Set the last python reference as a property of the C++ object + tester.obj = PyChild() + pytest.gc_collect() + + # python reference is still around since C++ has it now + assert tester.is_base_used() is False + assert tester.has_python_instance() is True + assert tester.obj.is_base_used() is False + assert tester.obj.has_python_instance() is True + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_arg_identity(): + import weakref + + tester = m.SpBaseTester() + + obj = m.SpBase() + objref = weakref.ref(obj) + + tester.set_object(obj) + del obj + pytest.gc_collect() + + # NOTE: the behavior below is DIFFERENT from PR #2839 + # python reference is gone because it is not an Alias instance + assert objref() is None + assert tester.has_python_instance() is False + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_alias_nonpython(): + tester = m.SpBaseTester() + + # C++ creates the object, a python instance shouldn't exist + tester.set_nonpython_instance() + assert tester.is_base_used() is True + assert tester.has_instance() is True + assert tester.has_python_instance() is False + + # Now a python instance exists + cobj = tester.get_object() + assert cobj.has_python_instance() + assert tester.has_instance() is True + assert tester.has_python_instance() is True + + # Now it's gone + del cobj + pytest.gc_collect() + assert tester.has_instance() is True + assert tester.has_python_instance() is False + + # When we pass it as an arg to a new tester the python instance should + # disappear because it wasn't created with an alias + new_tester = m.SpBaseTester() + + cobj = tester.get_object() + assert cobj.has_python_instance() + + new_tester.set_object(cobj) + assert tester.has_python_instance() is True + assert new_tester.has_python_instance() is True + + del cobj + pytest.gc_collect() + + # Gone! + assert tester.has_instance() is True + assert tester.has_python_instance() is False + assert new_tester.has_instance() is True + assert new_tester.has_python_instance() is False + + +@pytest.mark.skipif("env.GRAALPY", reason="Cannot reliably trigger GC") +def test_shared_ptr_goaway(): + import weakref + + tester = m.SpGoAwayTester() + + obj = m.SpGoAway() + objref = weakref.ref(obj) + + assert tester.obj is None + + tester.obj = obj + del obj + pytest.gc_collect() + + # python reference is no longer around + assert objref() is None + # C++ reference is still around + assert tester.obj is not None + + +def test_infinite(): + tester = m.SpBaseTester() + while True: + tester.set_object(m.SpBase()) + break # Comment out for manual leak checking (use `top` command). + + +@pytest.mark.parametrize( + "pass_through_func", [m.pass_through_shd_ptr, m.pass_through_shd_ptr_release_gil] +) +def test_std_make_shared_factory(pass_through_func): + class PyChild(m.SpBase): + def __init__(self): + super().__init__(0) + + obj = PyChild() + while True: + assert pass_through_func(obj) is obj + break # Comment out for manual leak checking (use `top` command). diff --git a/tests/test_class_sh_trampoline_unique_ptr.cpp b/tests/test_class_sh_trampoline_unique_ptr.cpp new file mode 100644 index 0000000000..1a58094201 --- /dev/null +++ b/tests/test_class_sh_trampoline_unique_ptr.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11/smart_holder.h" +#include "pybind11/trampoline_self_life_support.h" +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_trampoline_unique_ptr { + +class Class { +public: + virtual ~Class() = default; + + void setVal(std::uint64_t val) { val_ = val; } + std::uint64_t getVal() const { return val_; } + + virtual std::unique_ptr clone() const = 0; + virtual int foo() const = 0; + +protected: + Class() = default; + + // Some compilers complain about implicitly defined versions of some of the following: + Class(const Class &) = default; + +private: + std::uint64_t val_ = 0; +}; + +} // namespace class_sh_trampoline_unique_ptr +} // namespace pybind11_tests + +namespace pybind11_tests { +namespace class_sh_trampoline_unique_ptr { + +class PyClass : public Class, public py::trampoline_self_life_support { +public: + std::unique_ptr clone() const override { + PYBIND11_OVERRIDE_PURE(std::unique_ptr, Class, clone); + } + + int foo() const override { PYBIND11_OVERRIDE_PURE(int, Class, foo); } +}; + +} // namespace class_sh_trampoline_unique_ptr +} // namespace pybind11_tests + +TEST_SUBMODULE(class_sh_trampoline_unique_ptr, m) { + using namespace pybind11_tests::class_sh_trampoline_unique_ptr; + + py::classh(m, "Class") + .def(py::init<>()) + .def("set_val", &Class::setVal) + .def("get_val", &Class::getVal) + .def("clone", &Class::clone) + .def("foo", &Class::foo); + + m.def("clone", [](const Class &obj) { return obj.clone(); }); + m.def("clone_and_foo", [](const Class &obj) { return obj.clone()->foo(); }); +} diff --git a/tests/test_class_sh_trampoline_unique_ptr.py b/tests/test_class_sh_trampoline_unique_ptr.py new file mode 100644 index 0000000000..7799df6d61 --- /dev/null +++ b/tests/test_class_sh_trampoline_unique_ptr.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pybind11_tests.class_sh_trampoline_unique_ptr as m + + +class MyClass(m.Class): + def foo(self): + return 10 + self.get_val() + + def clone(self): + cloned = MyClass() + cloned.set_val(self.get_val() + 3) + return cloned + + +def test_m_clone(): + obj = MyClass() + while True: + obj.set_val(5) + obj = m.clone(obj) + assert obj.get_val() == 5 + 3 + assert obj.foo() == 10 + 5 + 3 + return # Comment out for manual leak checking (use `top` command). + + +def test_m_clone_and_foo(): + obj = MyClass() + obj.set_val(7) + while True: + assert m.clone_and_foo(obj) == 10 + 7 + 3 + return # Comment out for manual leak checking (use `top` command). diff --git a/tests/test_class_sh_unique_ptr_custom_deleter.cpp b/tests/test_class_sh_unique_ptr_custom_deleter.cpp new file mode 100644 index 0000000000..870779fdd8 --- /dev/null +++ b/tests/test_class_sh_unique_ptr_custom_deleter.cpp @@ -0,0 +1,32 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_unique_ptr_custom_deleter { + +// Reduced from a PyCLIF use case in the wild by @wangxf123456. +class Pet { +public: + using Ptr = std::unique_ptr>; + + std::string name; + + static Ptr New(const std::string &name) { + return Ptr(new Pet(name), std::default_delete()); + } + +private: + explicit Pet(const std::string &name) : name(name) {} +}; + +TEST_SUBMODULE(class_sh_unique_ptr_custom_deleter, m) { + py::classh(m, "Pet").def_readwrite("name", &Pet::name); + + m.def("create", &Pet::New); +} + +} // namespace class_sh_unique_ptr_custom_deleter +} // namespace pybind11_tests diff --git a/tests/test_class_sh_unique_ptr_custom_deleter.py b/tests/test_class_sh_unique_ptr_custom_deleter.py new file mode 100644 index 0000000000..34aa520682 --- /dev/null +++ b/tests/test_class_sh_unique_ptr_custom_deleter.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from pybind11_tests import class_sh_unique_ptr_custom_deleter as m + + +def test_create(): + pet = m.create("abc") + assert pet.name == "abc" diff --git a/tests/test_class_sh_unique_ptr_member.cpp b/tests/test_class_sh_unique_ptr_member.cpp new file mode 100644 index 0000000000..fc0a6a16e8 --- /dev/null +++ b/tests/test_class_sh_unique_ptr_member.cpp @@ -0,0 +1,52 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_unique_ptr_member { + +class pointee { // NOT copyable. +public: + pointee() = default; + + int get_int() const { return 213; } + + pointee(const pointee &) = delete; + pointee(pointee &&) = delete; + pointee &operator=(const pointee &) = delete; + pointee &operator=(pointee &&) = delete; +}; + +inline std::unique_ptr make_unique_pointee() { + return std::unique_ptr(new pointee); +} + +class ptr_owner { +public: + explicit ptr_owner(std::unique_ptr ptr) : ptr_(std::move(ptr)) {} + + bool is_owner() const { return bool(ptr_); } + + std::unique_ptr give_up_ownership_via_unique_ptr() { return std::move(ptr_); } + std::shared_ptr give_up_ownership_via_shared_ptr() { return std::move(ptr_); } + +private: + std::unique_ptr ptr_; +}; + +TEST_SUBMODULE(class_sh_unique_ptr_member, m) { + py::classh(m, "pointee").def(py::init<>()).def("get_int", &pointee::get_int); + + m.def("make_unique_pointee", make_unique_pointee); + + py::class_(m, "ptr_owner") + .def(py::init>(), py::arg("ptr")) + .def("is_owner", &ptr_owner::is_owner) + .def("give_up_ownership_via_unique_ptr", &ptr_owner::give_up_ownership_via_unique_ptr) + .def("give_up_ownership_via_shared_ptr", &ptr_owner::give_up_ownership_via_shared_ptr); +} + +} // namespace class_sh_unique_ptr_member +} // namespace pybind11_tests diff --git a/tests/test_class_sh_unique_ptr_member.py b/tests/test_class_sh_unique_ptr_member.py new file mode 100644 index 0000000000..a5d2ccd234 --- /dev/null +++ b/tests/test_class_sh_unique_ptr_member.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_unique_ptr_member as m + + +def test_make_unique_pointee(): + obj = m.make_unique_pointee() + assert obj.get_int() == 213 + + +@pytest.mark.parametrize( + "give_up_ownership_via", + ["give_up_ownership_via_unique_ptr", "give_up_ownership_via_shared_ptr"], +) +def test_pointee_and_ptr_owner(give_up_ownership_via): + obj = m.pointee() + assert obj.get_int() == 213 + owner = m.ptr_owner(obj) + with pytest.raises(ValueError, match="Python instance was disowned"): + obj.get_int() + assert owner.is_owner() + reclaimed = getattr(owner, give_up_ownership_via)() + assert not owner.is_owner() + assert reclaimed.get_int() == 213 diff --git a/tests/test_class_sh_virtual_py_cpp_mix.cpp b/tests/test_class_sh_virtual_py_cpp_mix.cpp new file mode 100644 index 0000000000..b39423eb45 --- /dev/null +++ b/tests/test_class_sh_virtual_py_cpp_mix.cpp @@ -0,0 +1,60 @@ +#include + +#include "pybind11_tests.h" + +#include + +namespace pybind11_tests { +namespace class_sh_virtual_py_cpp_mix { + +class Base { +public: + virtual ~Base() = default; + virtual int get() const { return 101; } + + // Some compilers complain about implicitly defined versions of some of the following: + Base() = default; + Base(const Base &) = default; +}; + +class CppDerivedPlain : public Base { +public: + int get() const override { return 202; } +}; + +class CppDerived : public Base { +public: + int get() const override { return 212; } +}; + +int get_from_cpp_plainc_ptr(const Base *b) { return b->get() + 4000; } + +int get_from_cpp_unique_ptr(std::unique_ptr b) { return b->get() + 5000; } + +struct BaseVirtualOverrider : Base, py::trampoline_self_life_support { + using Base::Base; + + int get() const override { PYBIND11_OVERRIDE(int, Base, get); } +}; + +struct CppDerivedVirtualOverrider : CppDerived, py::trampoline_self_life_support { + using CppDerived::CppDerived; + + int get() const override { PYBIND11_OVERRIDE(int, CppDerived, get); } +}; + +} // namespace class_sh_virtual_py_cpp_mix +} // namespace pybind11_tests + +using namespace pybind11_tests::class_sh_virtual_py_cpp_mix; + +TEST_SUBMODULE(class_sh_virtual_py_cpp_mix, m) { + py::classh(m, "Base").def(py::init<>()).def("get", &Base::get); + + py::classh(m, "CppDerivedPlain").def(py::init<>()); + + py::classh(m, "CppDerived").def(py::init<>()); + + m.def("get_from_cpp_plainc_ptr", get_from_cpp_plainc_ptr, py::arg("b")); + m.def("get_from_cpp_unique_ptr", get_from_cpp_unique_ptr, py::arg("b")); +} diff --git a/tests/test_class_sh_virtual_py_cpp_mix.py b/tests/test_class_sh_virtual_py_cpp_mix.py new file mode 100644 index 0000000000..33133eb889 --- /dev/null +++ b/tests/test_class_sh_virtual_py_cpp_mix.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import pytest + +from pybind11_tests import class_sh_virtual_py_cpp_mix as m + + +class PyBase(m.Base): # Avoiding name PyDerived, for more systematic naming. + def __init__(self): + m.Base.__init__(self) + + def get(self): + return 323 + + +class PyCppDerived(m.CppDerived): + def __init__(self): + m.CppDerived.__init__(self) + + def get(self): + return 434 + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 101), + (PyBase, 323), + (m.CppDerivedPlain, 202), + (m.CppDerived, 212), + (PyCppDerived, 434), + ], +) +def test_base_get(ctor, expected): + obj = ctor() + assert obj.get() == expected + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 4101), + (PyBase, 4323), + (m.CppDerivedPlain, 4202), + (m.CppDerived, 4212), + (PyCppDerived, 4434), + ], +) +def test_get_from_cpp_plainc_ptr(ctor, expected): + obj = ctor() + assert m.get_from_cpp_plainc_ptr(obj) == expected + + +@pytest.mark.parametrize( + ("ctor", "expected"), + [ + (m.Base, 5101), + (PyBase, 5323), + (m.CppDerivedPlain, 5202), + (m.CppDerived, 5212), + (PyCppDerived, 5434), + ], +) +def test_get_from_cpp_unique_ptr(ctor, expected): + obj = ctor() + assert m.get_from_cpp_unique_ptr(obj) == expected diff --git a/ubench/holder_comparison.cpp b/ubench/holder_comparison.cpp new file mode 100644 index 0000000000..da7e744c6b --- /dev/null +++ b/ubench/holder_comparison.cpp @@ -0,0 +1,49 @@ +#include + +#include "number_bucket.h" + +#include +#include + +namespace hc { // holder comparison + +using nb_up = pybind11_ubench::number_bucket<1>; +using nb_sp = pybind11_ubench::number_bucket<2>; +using nb_pu = pybind11_ubench::number_bucket<3>; +using nb_sh = pybind11_ubench::number_bucket<4>; + +namespace py = pybind11; + +template +void wrap_number_bucket(py::module m, const char *class_name) { + py::class_(m, class_name) + .def(py::init(), py::arg("data_size") = 0) + .def("sum", &WrappedType::sum) + .def("add", &WrappedType::add, py::arg("other")); +} + +template +class padded_unique_ptr { + std::unique_ptr ptr; + char padding[sizeof(py::smart_holder) - sizeof(std::unique_ptr)]; + +public: + padded_unique_ptr(T *p) : ptr(p) {} + T *get() { return ptr.get(); } +}; + +static_assert(sizeof(padded_unique_ptr) == sizeof(py::smart_holder), + "Unexpected sizeof mismatch."); + +} // namespace hc + +PYBIND11_DECLARE_HOLDER_TYPE(T, hc::padded_unique_ptr); + +PYBIND11_MODULE(pybind11_ubench_holder_comparison, m) { + using namespace hc; + wrap_number_bucket>(m, "number_bucket_up"); + wrap_number_bucket>(m, "number_bucket_sp"); + m.def("sizeof_smart_holder", []() { return sizeof(py::smart_holder); }); + wrap_number_bucket>(m, "number_bucket_pu"); + wrap_number_bucket(m, "number_bucket_sh"); +} diff --git a/ubench/holder_comparison.py b/ubench/holder_comparison.py new file mode 100644 index 0000000000..d18a123024 --- /dev/null +++ b/ubench/holder_comparison.py @@ -0,0 +1,144 @@ +"""Simple comparison of holder performances, relative to unique_ptr holder.""" + +# ruff: noqa +# This code has no unit tests. +# ruff cleanup deferred until the next time this code is actually used. + +import collections +import sys +import time +from typing import Any, Callable, Dict, List + +import pybind11_ubench_holder_comparison as m # type: ignore[import-not-found] + +number_bucket_pc = None + + +def pflush(*args: Any, **kwargs: Any) -> None: + print(*args, **kwargs) + # Using "file" here because it is the name of the built-in keyword argument. + file = kwargs.get("file", sys.stdout) # pylint: disable=redefined-builtin + file.flush() # file object must have a flush method. + + +def run(args: List[str]) -> None: + if not args: + size_exponent_min = 0 + size_exponent_max = 16 + size_exponent_step = 4 + call_repetitions_first_pass = 100 + call_repetitions_target_elapsed_secs = 0.1 + num_samples = 10 + selected_holder_type = "all" + else: + assert len(args) == 7, ( + "size_exponent_min size_exponent_max size_exponent_step" + " call_repetitions_first_pass call_repetitions_target_elapsed_secs" + " num_samples selected_holder_type" + ) + size_exponent_min = int(args[0]) + size_exponent_max = int(args[1]) + size_exponent_step = int(args[2]) + call_repetitions_first_pass = int(args[3]) + call_repetitions_target_elapsed_secs = float(args[4]) + num_samples = int(args[5]) + selected_holder_type = args[6] + pflush( + "command-line arguments:", + size_exponent_min, + size_exponent_max, + size_exponent_step, + call_repetitions_first_pass, + "%.3f" % call_repetitions_target_elapsed_secs, + num_samples, + selected_holder_type, + ) + pflush("sizeof_smart_holder:", m.sizeof_smart_holder()) + + def find_call_repetitions( + callable: Callable[[int], float], + time_delta_floor: float = 1.0e-6, + target_elapsed_secs_multiplier: float = 1.05, # Empirical. + target_elapsed_secs_tolerance: float = 0.05, + max_iterations: int = 100, + ) -> int: + td_target = ( + call_repetitions_target_elapsed_secs * target_elapsed_secs_multiplier + ) + crd = call_repetitions_first_pass + for _ in range(max_iterations): + td = callable(crd) + crd = max(1, int(td_target * crd / max(td, time_delta_floor))) + if abs(td - td_target) / td_target < target_elapsed_secs_tolerance: + return crd + raise RuntimeError("find_call_repetitions failure: max_iterations exceeded.") + + for size_exponent in range( + size_exponent_min, size_exponent_max + 1, size_exponent_step + ): + data_size = 2**size_exponent + pflush(data_size, "data_size") + ratios: Dict[str, List[float]] = collections.defaultdict(list) + call_repetitions = None + for _ in range(num_samples): + row_0 = None + for nb_label, nb_type in [ + ("up", m.number_bucket_up), + ("sp", m.number_bucket_sp), + ("pu", m.number_bucket_pu), + ("sh", m.number_bucket_sh), + ("pc", number_bucket_pc), + ]: + if nb_label == "pc" and nb_type is None: + continue + if selected_holder_type != "all" and nb_label != selected_holder_type: + continue + nb1 = nb_type(data_size) + nb2 = nb_type(data_size) + + def many_sum(call_repetitions: int) -> float: + assert int(round(nb1.sum())) == data_size + t0 = time.time() + for _ in range(call_repetitions): + nb1.sum() + return time.time() - t0 + + def many_add(call_repetitions: int) -> float: + assert nb1.add(nb2) == data_size + t0 = time.time() + for _ in range(call_repetitions): + nb1.add(nb2) + return time.time() - t0 + + if call_repetitions is None: + call_repetitions = find_call_repetitions(many_sum) + pflush(call_repetitions, "call_repetitions") + + td_sum = many_sum(call_repetitions) + td_add = many_add(call_repetitions) + row = [td_sum, td_add] + if row_0 is None: + pflush(" Sum Add ratS ratA") + row_0 = row + else: + for curr, prev in zip(row, row_0): # type: ignore[unreachable] + if prev: + rat = curr / prev + else: + rat = -1 + row.append(curr / prev) + ratios[nb_label + "_ratS"].append(row[-2]) + ratios[nb_label + "_ratA"].append(row[-1]) + pflush(nb_label, " ".join(["%.3f" % v for v in row])) + pflush(" Min Mean Max") + for key, rat in ratios.items(): + print( + key, + "{:5.3f} {:5.3f} {:5.3f}".format( + min(rat), sum(rat) / len(rat), max(rat) + ), + ) + + +if __name__ == "__main__": + run(args=sys.argv[1:]) diff --git a/ubench/holder_comparison_extract_sheet_data.py b/ubench/holder_comparison_extract_sheet_data.py new file mode 100644 index 0000000000..0676603066 --- /dev/null +++ b/ubench/holder_comparison_extract_sheet_data.py @@ -0,0 +1,71 @@ +"""Extract mean ratios from holder_comparison.py output.""" + +# ruff: noqa +# This code has no unit tests. +# ruff cleanup deferred until the next time this code is actually used. + +import sys +from typing import List, Optional + + +def run(args: List[str]) -> None: + assert len(args) == 1, "log_holder_comparison.txt" + + log_lines = open(args[0]).read().splitlines() + + for ratx in ("_ratS ", "_ratA "): + print(ratx) + header = None + header_row = None + data_row = None + data_row_buffer: List[List[str]] = [] + + def show() -> Optional[List[str]]: + if header_row: + if header is None: # type: ignore[unreachable] + print(",".join(header_row)) + else: + assert header == header_row + if data_row is not None: + print(",".join(data_row)) # type: ignore[unreachable] + data_row_buffer.append(data_row) + return header_row + + for line in log_lines: + if line.endswith(" data_size"): + header = show() + flds = line.split() + assert len(flds) == 2 + header_row = ["data_size"] + data_row = [flds[0]] + elif line.endswith(" call_repetitions"): + flds = line.split() + assert len(flds) == 2 + assert header_row is not None + assert data_row is not None + header_row.append("calls") + data_row.append(flds[0]) + header_row.append("up") + data_row.append("1.000") + elif line[2:].startswith(ratx): + flds = line.split() + assert len(flds) == 4 + assert header_row is not None + assert data_row is not None + header_row.append(line[:2]) + data_row.append(flds[2]) + show() + + assert header_row is not None + print("Scaled to last column:") + print(",".join(header_row)) + for data_row in data_row_buffer: + data_row_rescaled = data_row[:2] + unit = float(data_row[-1]) + for fld in data_row[2:]: + data_row_rescaled.append("%.3f" % (float(fld) / unit)) + print(",".join(data_row_rescaled)) + + +if __name__ == "__main__": + run(args=sys.argv[1:]) diff --git a/ubench/number_bucket.h b/ubench/number_bucket.h new file mode 100644 index 0000000000..fe04a1e56f --- /dev/null +++ b/ubench/number_bucket.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +namespace pybind11_ubench { + +template +struct number_bucket { + std::vector data; + + explicit number_bucket(std::size_t data_size = 0) : data(data_size, 1.0) {} + + double sum() const { + std::size_t n = 0; + double s = 0; + const double *a = &*data.begin(); + const double *e = &*data.end(); + while (a != e) { + s += *a++; + n++; + } + if (n != data.size()) { + std::cerr << "Internal consistency failure (sum)." << std::endl; + std::terminate(); + } + return s; + } + + std::size_t add(const number_bucket &other) { + if (other.data.size() != data.size()) { + std::cerr << "Incompatible data sizes (add)." << std::endl; + std::terminate(); + } + std::size_t n = 0; + double *a = &*data.begin(); + const double *e = &*data.end(); + const double *b = &*other.data.begin(); + while (a != e) { + *a++ += *b++; + n++; + } + return n; + } + +private: + number_bucket(const number_bucket &) = delete; + number_bucket(number_bucket &&) = delete; + number_bucket &operator=(const number_bucket &) = delete; + number_bucket &operator=(number_bucket &&) = delete; +}; + +} // namespace pybind11_ubench diff --git a/ubench/python/number_bucket.clif b/ubench/python/number_bucket.clif new file mode 100644 index 0000000000..ef704c0905 --- /dev/null +++ b/ubench/python/number_bucket.clif @@ -0,0 +1,6 @@ +from "pybind11/ubench/number_bucket.h": + namespace `pybind11_ubench`: + class `number_bucket<0>` as number_bucket_pc: + def __init__(self, data_size: int = default) + def sum(self) -> float + def add(self, other: number_bucket_pc) -> int From f01bc09dd6c67a9b6a797d73d17ff2ad45985040 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 22 Feb 2025 11:26:46 -0800 Subject: [PATCH 02/25] Remove ubench/ directory. --- MANIFEST.in | 1 - ubench/holder_comparison.cpp | 49 ------ ubench/holder_comparison.py | 144 ------------------ .../holder_comparison_extract_sheet_data.py | 71 --------- ubench/number_bucket.h | 55 ------- ubench/python/number_bucket.clif | 6 - 6 files changed, 326 deletions(-) delete mode 100644 ubench/holder_comparison.cpp delete mode 100644 ubench/holder_comparison.py delete mode 100644 ubench/holder_comparison_extract_sheet_data.py delete mode 100644 ubench/number_bucket.h delete mode 100644 ubench/python/number_bucket.clif diff --git a/MANIFEST.in b/MANIFEST.in index ff9ee6c74f..c13394558f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ prune tests -prune ubench include README_smart_holder.rst recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py diff --git a/ubench/holder_comparison.cpp b/ubench/holder_comparison.cpp deleted file mode 100644 index da7e744c6b..0000000000 --- a/ubench/holder_comparison.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include - -#include "number_bucket.h" - -#include -#include - -namespace hc { // holder comparison - -using nb_up = pybind11_ubench::number_bucket<1>; -using nb_sp = pybind11_ubench::number_bucket<2>; -using nb_pu = pybind11_ubench::number_bucket<3>; -using nb_sh = pybind11_ubench::number_bucket<4>; - -namespace py = pybind11; - -template -void wrap_number_bucket(py::module m, const char *class_name) { - py::class_(m, class_name) - .def(py::init(), py::arg("data_size") = 0) - .def("sum", &WrappedType::sum) - .def("add", &WrappedType::add, py::arg("other")); -} - -template -class padded_unique_ptr { - std::unique_ptr ptr; - char padding[sizeof(py::smart_holder) - sizeof(std::unique_ptr)]; - -public: - padded_unique_ptr(T *p) : ptr(p) {} - T *get() { return ptr.get(); } -}; - -static_assert(sizeof(padded_unique_ptr) == sizeof(py::smart_holder), - "Unexpected sizeof mismatch."); - -} // namespace hc - -PYBIND11_DECLARE_HOLDER_TYPE(T, hc::padded_unique_ptr); - -PYBIND11_MODULE(pybind11_ubench_holder_comparison, m) { - using namespace hc; - wrap_number_bucket>(m, "number_bucket_up"); - wrap_number_bucket>(m, "number_bucket_sp"); - m.def("sizeof_smart_holder", []() { return sizeof(py::smart_holder); }); - wrap_number_bucket>(m, "number_bucket_pu"); - wrap_number_bucket(m, "number_bucket_sh"); -} diff --git a/ubench/holder_comparison.py b/ubench/holder_comparison.py deleted file mode 100644 index d18a123024..0000000000 --- a/ubench/holder_comparison.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Simple comparison of holder performances, relative to unique_ptr holder.""" - -# ruff: noqa -# This code has no unit tests. -# ruff cleanup deferred until the next time this code is actually used. - -import collections -import sys -import time -from typing import Any, Callable, Dict, List - -import pybind11_ubench_holder_comparison as m # type: ignore[import-not-found] - -number_bucket_pc = None - - -def pflush(*args: Any, **kwargs: Any) -> None: - print(*args, **kwargs) - # Using "file" here because it is the name of the built-in keyword argument. - file = kwargs.get("file", sys.stdout) # pylint: disable=redefined-builtin - file.flush() # file object must have a flush method. - - -def run(args: List[str]) -> None: - if not args: - size_exponent_min = 0 - size_exponent_max = 16 - size_exponent_step = 4 - call_repetitions_first_pass = 100 - call_repetitions_target_elapsed_secs = 0.1 - num_samples = 10 - selected_holder_type = "all" - else: - assert len(args) == 7, ( - "size_exponent_min size_exponent_max size_exponent_step" - " call_repetitions_first_pass call_repetitions_target_elapsed_secs" - " num_samples selected_holder_type" - ) - size_exponent_min = int(args[0]) - size_exponent_max = int(args[1]) - size_exponent_step = int(args[2]) - call_repetitions_first_pass = int(args[3]) - call_repetitions_target_elapsed_secs = float(args[4]) - num_samples = int(args[5]) - selected_holder_type = args[6] - pflush( - "command-line arguments:", - size_exponent_min, - size_exponent_max, - size_exponent_step, - call_repetitions_first_pass, - "%.3f" % call_repetitions_target_elapsed_secs, - num_samples, - selected_holder_type, - ) - pflush("sizeof_smart_holder:", m.sizeof_smart_holder()) - - def find_call_repetitions( - callable: Callable[[int], float], - time_delta_floor: float = 1.0e-6, - target_elapsed_secs_multiplier: float = 1.05, # Empirical. - target_elapsed_secs_tolerance: float = 0.05, - max_iterations: int = 100, - ) -> int: - td_target = ( - call_repetitions_target_elapsed_secs * target_elapsed_secs_multiplier - ) - crd = call_repetitions_first_pass - for _ in range(max_iterations): - td = callable(crd) - crd = max(1, int(td_target * crd / max(td, time_delta_floor))) - if abs(td - td_target) / td_target < target_elapsed_secs_tolerance: - return crd - raise RuntimeError("find_call_repetitions failure: max_iterations exceeded.") - - for size_exponent in range( - size_exponent_min, size_exponent_max + 1, size_exponent_step - ): - data_size = 2**size_exponent - pflush(data_size, "data_size") - ratios: Dict[str, List[float]] = collections.defaultdict(list) - call_repetitions = None - for _ in range(num_samples): - row_0 = None - for nb_label, nb_type in [ - ("up", m.number_bucket_up), - ("sp", m.number_bucket_sp), - ("pu", m.number_bucket_pu), - ("sh", m.number_bucket_sh), - ("pc", number_bucket_pc), - ]: - if nb_label == "pc" and nb_type is None: - continue - if selected_holder_type != "all" and nb_label != selected_holder_type: - continue - nb1 = nb_type(data_size) - nb2 = nb_type(data_size) - - def many_sum(call_repetitions: int) -> float: - assert int(round(nb1.sum())) == data_size - t0 = time.time() - for _ in range(call_repetitions): - nb1.sum() - return time.time() - t0 - - def many_add(call_repetitions: int) -> float: - assert nb1.add(nb2) == data_size - t0 = time.time() - for _ in range(call_repetitions): - nb1.add(nb2) - return time.time() - t0 - - if call_repetitions is None: - call_repetitions = find_call_repetitions(many_sum) - pflush(call_repetitions, "call_repetitions") - - td_sum = many_sum(call_repetitions) - td_add = many_add(call_repetitions) - row = [td_sum, td_add] - if row_0 is None: - pflush(" Sum Add ratS ratA") - row_0 = row - else: - for curr, prev in zip(row, row_0): # type: ignore[unreachable] - if prev: - rat = curr / prev - else: - rat = -1 - row.append(curr / prev) - ratios[nb_label + "_ratS"].append(row[-2]) - ratios[nb_label + "_ratA"].append(row[-1]) - pflush(nb_label, " ".join(["%.3f" % v for v in row])) - pflush(" Min Mean Max") - for key, rat in ratios.items(): - print( - key, - "{:5.3f} {:5.3f} {:5.3f}".format( - min(rat), sum(rat) / len(rat), max(rat) - ), - ) - - -if __name__ == "__main__": - run(args=sys.argv[1:]) diff --git a/ubench/holder_comparison_extract_sheet_data.py b/ubench/holder_comparison_extract_sheet_data.py deleted file mode 100644 index 0676603066..0000000000 --- a/ubench/holder_comparison_extract_sheet_data.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Extract mean ratios from holder_comparison.py output.""" - -# ruff: noqa -# This code has no unit tests. -# ruff cleanup deferred until the next time this code is actually used. - -import sys -from typing import List, Optional - - -def run(args: List[str]) -> None: - assert len(args) == 1, "log_holder_comparison.txt" - - log_lines = open(args[0]).read().splitlines() - - for ratx in ("_ratS ", "_ratA "): - print(ratx) - header = None - header_row = None - data_row = None - data_row_buffer: List[List[str]] = [] - - def show() -> Optional[List[str]]: - if header_row: - if header is None: # type: ignore[unreachable] - print(",".join(header_row)) - else: - assert header == header_row - if data_row is not None: - print(",".join(data_row)) # type: ignore[unreachable] - data_row_buffer.append(data_row) - return header_row - - for line in log_lines: - if line.endswith(" data_size"): - header = show() - flds = line.split() - assert len(flds) == 2 - header_row = ["data_size"] - data_row = [flds[0]] - elif line.endswith(" call_repetitions"): - flds = line.split() - assert len(flds) == 2 - assert header_row is not None - assert data_row is not None - header_row.append("calls") - data_row.append(flds[0]) - header_row.append("up") - data_row.append("1.000") - elif line[2:].startswith(ratx): - flds = line.split() - assert len(flds) == 4 - assert header_row is not None - assert data_row is not None - header_row.append(line[:2]) - data_row.append(flds[2]) - show() - - assert header_row is not None - print("Scaled to last column:") - print(",".join(header_row)) - for data_row in data_row_buffer: - data_row_rescaled = data_row[:2] - unit = float(data_row[-1]) - for fld in data_row[2:]: - data_row_rescaled.append("%.3f" % (float(fld) / unit)) - print(",".join(data_row_rescaled)) - - -if __name__ == "__main__": - run(args=sys.argv[1:]) diff --git a/ubench/number_bucket.h b/ubench/number_bucket.h deleted file mode 100644 index fe04a1e56f..0000000000 --- a/ubench/number_bucket.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace pybind11_ubench { - -template -struct number_bucket { - std::vector data; - - explicit number_bucket(std::size_t data_size = 0) : data(data_size, 1.0) {} - - double sum() const { - std::size_t n = 0; - double s = 0; - const double *a = &*data.begin(); - const double *e = &*data.end(); - while (a != e) { - s += *a++; - n++; - } - if (n != data.size()) { - std::cerr << "Internal consistency failure (sum)." << std::endl; - std::terminate(); - } - return s; - } - - std::size_t add(const number_bucket &other) { - if (other.data.size() != data.size()) { - std::cerr << "Incompatible data sizes (add)." << std::endl; - std::terminate(); - } - std::size_t n = 0; - double *a = &*data.begin(); - const double *e = &*data.end(); - const double *b = &*other.data.begin(); - while (a != e) { - *a++ += *b++; - n++; - } - return n; - } - -private: - number_bucket(const number_bucket &) = delete; - number_bucket(number_bucket &&) = delete; - number_bucket &operator=(const number_bucket &) = delete; - number_bucket &operator=(number_bucket &&) = delete; -}; - -} // namespace pybind11_ubench diff --git a/ubench/python/number_bucket.clif b/ubench/python/number_bucket.clif deleted file mode 100644 index ef704c0905..0000000000 --- a/ubench/python/number_bucket.clif +++ /dev/null @@ -1,6 +0,0 @@ -from "pybind11/ubench/number_bucket.h": - namespace `pybind11_ubench`: - class `number_bucket<0>` as number_bucket_pc: - def __init__(self, data_size: int = default) - def sum(self) -> float - def add(self, other: number_bucket_pc) -> int From 4d8973ec91202fc94bbbb7ea21b8b4d9cf84abc0 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 22 Feb 2025 13:28:13 -0800 Subject: [PATCH 03/25] Remove include/pybind11/smart_holder.h --- CMakeLists.txt | 1 - include/pybind11/smart_holder.h | 14 -------------- tests/extra_python_package/test_files.py | 1 - tests/test_class_sh_basic.cpp | 2 -- tests/test_class_sh_disowning.cpp | 2 -- tests/test_class_sh_disowning_mi.cpp | 2 -- tests/test_class_sh_factory_constructors.cpp | 2 -- tests/test_class_sh_inheritance.cpp | 2 -- tests/test_class_sh_mi_thunks.cpp | 3 --- tests/test_class_sh_property.cpp | 1 - tests/test_class_sh_property_non_owning.cpp | 1 - tests/test_class_sh_shared_ptr_copy_move.cpp | 2 -- tests/test_class_sh_trampoline_basic.cpp | 2 -- .../test_class_sh_trampoline_self_life_support.cpp | 1 - .../test_class_sh_trampoline_shared_from_this.cpp | 1 - ...test_class_sh_trampoline_shared_ptr_cpp_arg.cpp | 1 - tests/test_class_sh_trampoline_unique_ptr.cpp | 1 - tests/test_class_sh_unique_ptr_custom_deleter.cpp | 2 -- tests/test_class_sh_unique_ptr_member.cpp | 2 -- tests/test_class_sh_virtual_py_cpp_mix.cpp | 2 -- 20 files changed, 45 deletions(-) delete mode 100644 include/pybind11/smart_holder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d8323c10f8..8783dba6e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,7 +164,6 @@ set(PYBIND11_HEADERS include/pybind11/operators.h include/pybind11/pybind11.h include/pybind11/pytypes.h - include/pybind11/smart_holder.h include/pybind11/stl.h include/pybind11/stl_bind.h include/pybind11/stl/filesystem.h diff --git a/include/pybind11/smart_holder.h b/include/pybind11/smart_holder.h deleted file mode 100644 index 5f568a5529..0000000000 --- a/include/pybind11/smart_holder.h +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2021-2024 The Pybind Development Team. -// All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -#pragma once - -#include "pybind11.h" - -// Legacy macros introduced with smart_holder_type_casters implementation in 2021. -// Deprecated. -#define PYBIND11_TYPE_CASTER_BASE_HOLDER(...) -#define PYBIND11_SMART_HOLDER_TYPE_CASTERS(...) -#define PYBIND11_SH_AVL(...) // "Smart_Holder if AVaiLable" -#define PYBIND11_SH_DEF(...) // "Smart_Holder if DEFault" diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index f50fb15304..4cd4bedfd8 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -43,7 +43,6 @@ "include/pybind11/options.h", "include/pybind11/pybind11.h", "include/pybind11/pytypes.h", - "include/pybind11/smart_holder.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", "include/pybind11/trampoline_self_life_support.h", diff --git a/tests/test_class_sh_basic.cpp b/tests/test_class_sh_basic.cpp index ee973c2732..b89372b198 100644 --- a/tests/test_class_sh_basic.cpp +++ b/tests/test_class_sh_basic.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_disowning.cpp b/tests/test_class_sh_disowning.cpp index 97c95287be..490e6d59f2 100644 --- a/tests/test_class_sh_disowning.cpp +++ b/tests/test_class_sh_disowning.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_disowning_mi.cpp b/tests/test_class_sh_disowning_mi.cpp index 8b96672536..d0ffd45ec0 100644 --- a/tests/test_class_sh_disowning_mi.cpp +++ b/tests/test_class_sh_disowning_mi.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_factory_constructors.cpp b/tests/test_class_sh_factory_constructors.cpp index 3499eaea5d..b3a8daea51 100644 --- a/tests/test_class_sh_factory_constructors.cpp +++ b/tests/test_class_sh_factory_constructors.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_inheritance.cpp b/tests/test_class_sh_inheritance.cpp index bb46fb92e0..8bdd0a7f88 100644 --- a/tests/test_class_sh_inheritance.cpp +++ b/tests/test_class_sh_inheritance.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_mi_thunks.cpp b/tests/test_class_sh_mi_thunks.cpp index 244ccbfacb..d8548ec5c6 100644 --- a/tests/test_class_sh_mi_thunks.cpp +++ b/tests/test_class_sh_mi_thunks.cpp @@ -1,6 +1,3 @@ -#include -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_property.cpp b/tests/test_class_sh_property.cpp index 6476a9cad8..8863ad7d7b 100644 --- a/tests/test_class_sh_property.cpp +++ b/tests/test_class_sh_property.cpp @@ -3,7 +3,6 @@ // valu(e), ref(erence), ptr or p (pointer), r = rvalue, m = mutable, c = const, // sh = shared_ptr, uq = unique_ptr. -#include "pybind11/smart_holder.h" #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_property_non_owning.cpp b/tests/test_class_sh_property_non_owning.cpp index 42f1f6fd35..45fe7c7beb 100644 --- a/tests/test_class_sh_property_non_owning.cpp +++ b/tests/test_class_sh_property_non_owning.cpp @@ -1,4 +1,3 @@ -#include "pybind11/smart_holder.h" #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_shared_ptr_copy_move.cpp b/tests/test_class_sh_shared_ptr_copy_move.cpp index 5eb0ef3edb..889425a0b0 100644 --- a/tests/test_class_sh_shared_ptr_copy_move.cpp +++ b/tests/test_class_sh_shared_ptr_copy_move.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_trampoline_basic.cpp b/tests/test_class_sh_trampoline_basic.cpp index 7ae3be5bf9..0f42dcc717 100644 --- a/tests/test_class_sh_trampoline_basic.cpp +++ b/tests/test_class_sh_trampoline_basic.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_trampoline_self_life_support.cpp b/tests/test_class_sh_trampoline_self_life_support.cpp index 6c14e39c99..22b728e28c 100644 --- a/tests/test_class_sh_trampoline_self_life_support.cpp +++ b/tests/test_class_sh_trampoline_self_life_support.cpp @@ -2,7 +2,6 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#include "pybind11/smart_holder.h" #include "pybind11/trampoline_self_life_support.h" #include "pybind11_tests.h" diff --git a/tests/test_class_sh_trampoline_shared_from_this.cpp b/tests/test_class_sh_trampoline_shared_from_this.cpp index 9ccc213298..dc6bf1c72a 100644 --- a/tests/test_class_sh_trampoline_shared_from_this.cpp +++ b/tests/test_class_sh_trampoline_shared_from_this.cpp @@ -2,7 +2,6 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#include "pybind11/smart_holder.h" #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp index ee381f2eb8..49e1ac885d 100644 --- a/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp +++ b/tests/test_class_sh_trampoline_shared_ptr_cpp_arg.cpp @@ -2,7 +2,6 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#include "pybind11/smart_holder.h" #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_trampoline_unique_ptr.cpp b/tests/test_class_sh_trampoline_unique_ptr.cpp index 1a58094201..debe3324eb 100644 --- a/tests/test_class_sh_trampoline_unique_ptr.cpp +++ b/tests/test_class_sh_trampoline_unique_ptr.cpp @@ -2,7 +2,6 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#include "pybind11/smart_holder.h" #include "pybind11/trampoline_self_life_support.h" #include "pybind11_tests.h" diff --git a/tests/test_class_sh_unique_ptr_custom_deleter.cpp b/tests/test_class_sh_unique_ptr_custom_deleter.cpp index 870779fdd8..adaa2e47d0 100644 --- a/tests/test_class_sh_unique_ptr_custom_deleter.cpp +++ b/tests/test_class_sh_unique_ptr_custom_deleter.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_unique_ptr_member.cpp b/tests/test_class_sh_unique_ptr_member.cpp index fc0a6a16e8..50c505a655 100644 --- a/tests/test_class_sh_unique_ptr_member.cpp +++ b/tests/test_class_sh_unique_ptr_member.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include diff --git a/tests/test_class_sh_virtual_py_cpp_mix.cpp b/tests/test_class_sh_virtual_py_cpp_mix.cpp index b39423eb45..df8af19e48 100644 --- a/tests/test_class_sh_virtual_py_cpp_mix.cpp +++ b/tests/test_class_sh_virtual_py_cpp_mix.cpp @@ -1,5 +1,3 @@ -#include - #include "pybind11_tests.h" #include From c807ec3f11c7653f70b88a01904d8d90854ad973 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 22 Feb 2025 23:04:29 -0800 Subject: [PATCH 04/25] [ci skip] smart_ptrs.rst updates [WIP/unfinished] --- docs/advanced/smart_ptrs.rst | 232 ++++++++++++++++++++++------------- 1 file changed, 147 insertions(+), 85 deletions(-) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index e2e9f3f37f..5b5531f75f 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -1,131 +1,124 @@ -Smart pointers -############## +Smart pointers & ``py::class_`` +############################### -.. Note:: +The binding generator for classes, ``py::class_``, can be passed a template +type that denotes a special *holder* type that is used to manage references to +the object. If no such holder type template argument is given, the default for +a type ``T`` is ``std::unique_ptr``. - This is the pybind11 **smart_holder** branch, but the information - below has NOT been updated accordingly yet. Please refer to - ``README_smart_holder.rst`` under the top-level pybind11 directory - for updated information about smart pointers. +``py::smart_holder`` +==================== -std::unique_ptr -=============== +Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is +the recommended ``py::class_`` holder for all situations, but it is **not** +the default holder, and there is no intent to make it the default holder in +the future, based on the assumption that this would cause more disruption +than it is worth. -Given a class ``Example`` with Python bindings, it's possible to return -instances wrapped in C++11 unique pointers, like so +It is extremely easy to change existing pybind11 client code to use the safer +and more versatile ``py::smart_holder``. For a given C++ type ``T``, simply +change -.. code-block:: cpp +* ``py::class_`` to +* ``py::classh`` - std::unique_ptr create_example() { return std::unique_ptr(new Example()); } +.. note:: -.. code-block:: cpp + ``py::classh`` is simply a shortcut for ``py::class_``. - m.def("create_example", &create_example); +The ``py::classh`` functionality includes -In other words, there is nothing special that needs to be done. While returning -unique pointers in this way is allowed, it is *illegal* to use them as function -arguments. For instance, the following function signature cannot be processed -by pybind11. +* support for **two-way** Python/C++ conversions for both + ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. + — In contrast, ``py::class_`` only supports one-way C++-to-Python + conversions for ``std::unique_ptr``, or alternatively two-way + Python/C++ conversions for ``std::shared_ptr``, which then excludes + the one-way C++-to-Python ``std::unique_ptr`` conversions (this manifests + itself through undefined runtime behavior, often a segmentation fault + or double free). -.. code-block:: cpp +* passing a Python object back to C++ via ``std::unique_ptr``, safely + **disowning** the Python object. - void do_something_with_example(std::unique_ptr ex) { ... } +* safely passing `"trampoline" + `_ + objects (objects with C++ virtual function overrides implemented in + Python) via ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: + associated Python objects are automatically kept alive for the lifetime + of the smart-pointer. -The above signature would imply that Python needs to give up ownership of an -object that is passed to this function, which is generally not possible (for -instance, the object might be referenced elsewhere). +TODO(rwgk): Move to classes.rst -std::shared_ptr -=============== - -The binding generator for classes, :class:`class_`, can be passed a template -type that denotes a special *holder* type that is used to manage references to -the object. If no such holder type template argument is given, the default for -a type named ``Type`` is ``std::unique_ptr``, which means that the object -is deallocated when Python's reference count goes to zero. - -It is possible to switch to other types of reference counting wrappers or smart -pointers, which is useful in codebases that rely on them. For instance, the -following snippet causes ``std::shared_ptr`` to be used instead. +A pybind11 `"trampoline" +`_ +is a C++ helper class with virtual function overrides that transparently +call back from C++ into Python. To enable safely passing a ``std::unique_ptr`` +to a trampoline object between Python and C++, the trampoline class must +inherit from ``py::trampoline_self_life_support``, for example: .. code-block:: cpp - py::class_ /* <- holder type */> obj(m, "Example"); + class PyAnimal : public Animal, public py::trampoline_self_life_support { + ... + }; -Note that any particular class can only be associated with a single holder type. +A fairly minimal but complete example is :file:`tests/test_class_sh_trampoline_unique_ptr.cpp`. -One potential stumbling block when using holder types is that they need to be -applied consistently. Can you guess what's broken about the following binding -code? -.. code-block:: cpp +``std::unique_ptr`` +=================== - class Child { }; +This is the default ``py::class_`` holder and works as expected in most +situations. However, note that the handling of base-and-derived classes +involves a ``reinterpret_cast`` that has strictly speaking undefined +behavior. Also note that the ``std::unique_ptr`` holder only support passing +a ``std::unique_ptr`` from C++ to Python, but not the other way around. For +example, this code will work as expected when using ``py::class_``: - class Parent { - public: - Parent() : child(std::make_shared()) { } - Child *get_child() { return child.get(); } /* Hint: ** DON'T DO THIS ** */ - private: - std::shared_ptr child; - }; +.. code-block:: cpp - PYBIND11_MODULE(example, m) { - py::class_>(m, "Child"); + std::unique_ptr create_example() { return std::unique_ptr(new Example()); } - py::class_>(m, "Parent") - .def(py::init<>()) - .def("get_child", &Parent::get_child); - } +.. code-block:: cpp -The following Python code will cause undefined behavior (and likely a -segmentation fault). + m.def("create_example", &create_example); -.. code-block:: python +However, this will fail with ``py::class_`` (but work with +``py::classh``): - from example import Parent +.. code-block:: cpp - print(Parent().get_child()) + void do_something_with_example(std::unique_ptr ex) { ... } -The problem is that ``Parent::get_child()`` returns a pointer to an instance of -``Child``, but the fact that this instance is already managed by -``std::shared_ptr<...>`` is lost when passing raw pointers. In this case, -pybind11 will create a second independent ``std::shared_ptr<...>`` that also -claims ownership of the pointer. In the end, the object will be freed **twice** -since these shared pointers have no way of knowing about each other. -There are two ways to resolve this issue: +``std::shared_ptr`` +=================== -1. For types that are managed by a smart pointer class, never use raw pointers - in function arguments or return values. In other words: always consistently - wrap pointers into their designated holder types (such as - ``std::shared_ptr<...>``). In this case, the signature of ``get_child()`` - should be modified as follows: +It is possible to use ``std::shared_ptr`` as the holder, for example: .. code-block:: cpp - std::shared_ptr get_child() { return child; } + py::class_ /* <- holder type */>(m, "Example"); -2. Adjust the definition of ``Child`` by specifying - ``std::enable_shared_from_this`` (see cppreference_ for details) as a - base class. This adds a small bit of information to ``Child`` that allows - pybind11 to realize that there is already an existing - ``std::shared_ptr<...>`` and communicate with it. In this case, the - declaration of ``Child`` should look as follows: +Compared to using ``py::classh``, there are two noteworthy disadvantages: -.. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this +* A ``py::class_`` for any particular C++ type ``T`` (and all its derived types) + can only use a single holder type. Therefore, ``std::unique_ptr`` + cannot even be passed from C++ to Python if the ``std::shared_ptr`` holder + is used. This will become apparent only at runtime, often through a + segmentation fault or double free. -.. code-block:: cpp +* Similar to the ``std::unique_ptr`` holder, the handling of base-and-derived + classes involves a ``reinterpret_cast`` that has strictly speaking undefined + behavior, although it works as expected in most situations. - class Child : public std::enable_shared_from_this { }; .. _smart_pointers: Custom smart pointers ===================== -pybind11 supports ``std::unique_ptr`` and ``std::shared_ptr`` right out of the -box. For any other custom smart pointer, transparent conversions can be enabled +For custom smart pointer, transparent conversions can be enabled using a macro invocation similar to the following. It must be declared at the top namespace level before any binding code: @@ -179,3 +172,72 @@ provides ``.get()`` functionality via ``.getPointer()``. The file :file:`tests/test_smart_ptr.cpp` contains a complete example that demonstrates how to work with custom reference-counting holder types in more detail. + + +Be careful to not undermine automatic lifetime management +========================================================= + +One potential stumbling block when using holder types is that they need to be +applied consistently. Can you guess what's broken about the following binding +code? + +.. code-block:: cpp + + class Child { }; + + class Parent { + public: + Parent() : child(std::make_shared()) { } + Child *get_child() { return child.get(); } /* Hint: ** DON'T DO THIS ** */ + private: + std::shared_ptr child; + }; + + PYBIND11_MODULE(example, m) { + py::class_>(m, "Child"); + + py::class_>(m, "Parent") + .def(py::init<>()) + .def("get_child", &Parent::get_child); + } + +The following Python code will cause undefined behavior (and likely a +segmentation fault). + +.. code-block:: python + + from example import Parent + + print(Parent().get_child()) + +The problem is that ``Parent::get_child()`` returns a pointer to an instance of +``Child``, but the fact that this instance is already managed by +``std::shared_ptr<...>`` is lost when passing raw pointers. In this case, +pybind11 will create a second independent ``std::shared_ptr<...>`` that also +claims ownership of the pointer. In the end, the object will be freed **twice** +since these shared pointers have no way of knowing about each other. + +There are two ways to resolve this issue: + +1. For types that are managed by a smart pointer class, never use raw pointers + in function arguments or return values. In other words: always consistently + wrap pointers into their designated holder types (such as + ``std::shared_ptr<...>``). In this case, the signature of ``get_child()`` + should be modified as follows: + +.. code-block:: cpp + + std::shared_ptr get_child() { return child; } + +2. Adjust the definition of ``Child`` by specifying + ``std::enable_shared_from_this`` (see cppreference_ for details) as a + base class. This adds a small bit of information to ``Child`` that allows + pybind11 to realize that there is already an existing + ``std::shared_ptr<...>`` and communicate with it. In this case, the + declaration of ``Child`` should look as follows: + +.. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this + +.. code-block:: cpp + + class Child : public std::enable_shared_from_this { }; From 76f4da3ca85a10767d7f3d578f1b2d3e0a03c7dc Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 11:14:47 -0800 Subject: [PATCH 05/25] [ci skip] smart_ptrs.rst updates continued; also updating classes.rst, advanced/classes.rst --- docs/advanced/classes.rst | 26 ++++++ docs/advanced/smart_ptrs.rst | 156 +++++++++++++++-------------------- docs/classes.rst | 10 ++- 3 files changed, 101 insertions(+), 91 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 1e221072f2..45c038ef7a 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -125,6 +125,32 @@ extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the necessary steps required to providing proper overriding support for inherited classes. +To enable safely passing a ``std::unique_ptr`` to a trampoline object between +Python and C++, + +1. the C++ type (``Animal`` above) must be wrapped with ``py::classh`` + (see :ref:`smart_holder`), and + +2. the trampoline helper class must inherit from + ``py::trampoline_self_life_support``. + +I.e. the example above needs these two changes: + +.. code-block:: cpp + + class PyAnimal : public Animal, public py::trampoline_self_life_support { + ... + }; + +.. code-block:: cpp + + py::classh(m, "Animal"); + +.. seealso:: + + A fairly minimal but complete example is in + :file:`tests/test_class_sh_trampoline_unique_ptr.cpp`. + The Python session below shows how to override ``Animal::go`` and invoke it via a virtual method call. diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 5b5531f75f..35ebd82e07 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -3,66 +3,52 @@ Smart pointers & ``py::class_`` The binding generator for classes, ``py::class_``, can be passed a template type that denotes a special *holder* type that is used to manage references to -the object. If no such holder type template argument is given, the default for +the object. If no such holder type template argument is given, the default for a type ``T`` is ``std::unique_ptr``. +.. note:: + + A ``py::class_`` for a given C++ type ``T`` — and all its derived types — + can only use a single holder type. + + +.. _smart_holder: + ``py::smart_holder`` ==================== Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is -the recommended ``py::class_`` holder for all situations, but it is **not** -the default holder, and there is no intent to make it the default holder in -the future, based on the assumption that this would cause more disruption -than it is worth. - -It is extremely easy to change existing pybind11 client code to use the safer -and more versatile ``py::smart_holder``. For a given C++ type ``T``, simply -change +the recommended ``py::class_`` holder for most situations, but it is **not** +the default holder, and there are no plans to make it the default holder in +the future. This is based on the assumption that such a change would cause +more disruption than it is worth, especially because it is extremely easy +to use the safer and more versatile ``py::smart_holder``. For a given C++ +type ``T``, simply change * ``py::class_`` to -* ``py::classh`` + +* ``py::classh``. .. note:: - ``py::classh`` is simply a shortcut for ``py::class_``. + ``py::classh`` is a shortcut for ``py::class_``. -The ``py::classh`` functionality includes +The ``py::classh`` functionality includes the following: -* support for **two-way** Python/C++ conversions for both +* Support for **two-way** Python/C++ conversions for both ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. - — In contrast, ``py::class_`` only supports one-way C++-to-Python - conversions for ``std::unique_ptr``, or alternatively two-way - Python/C++ conversions for ``std::shared_ptr``, which then excludes - the one-way C++-to-Python ``std::unique_ptr`` conversions (this manifests - itself through undefined runtime behavior, often a segmentation fault - or double free). - -* passing a Python object back to C++ via ``std::unique_ptr``, safely + +* Passing a Python object back to C++ via ``std::unique_ptr``, safely **disowning** the Python object. -* safely passing `"trampoline" - `_ - objects (objects with C++ virtual function overrides implemented in - Python) via ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: +* Safely passing "trampoline" objects (objects with C++ virtual function + overrides implemented in Python, see :ref:`overriding_virtuals`) via + ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: associated Python objects are automatically kept alive for the lifetime of the smart-pointer. -TODO(rwgk): Move to classes.rst - -A pybind11 `"trampoline" -`_ -is a C++ helper class with virtual function overrides that transparently -call back from C++ into Python. To enable safely passing a ``std::unique_ptr`` -to a trampoline object between Python and C++, the trampoline class must -inherit from ``py::trampoline_self_life_support``, for example: - -.. code-block:: cpp - - class PyAnimal : public Animal, public py::trampoline_self_life_support { - ... - }; - -A fairly minimal but complete example is :file:`tests/test_class_sh_trampoline_unique_ptr.cpp`. +* Full support for ``std::enable_shared_from_this`` (`cppreference + `_). ``std::unique_ptr`` @@ -71,7 +57,7 @@ A fairly minimal but complete example is :file:`tests/test_class_sh_trampoline_u This is the default ``py::class_`` holder and works as expected in most situations. However, note that the handling of base-and-derived classes involves a ``reinterpret_cast`` that has strictly speaking undefined -behavior. Also note that the ``std::unique_ptr`` holder only support passing +behavior. Also note that the ``std::unique_ptr`` holder only supports passing a ``std::unique_ptr`` from C++ to Python, but not the other way around. For example, this code will work as expected when using ``py::class_``: @@ -83,7 +69,7 @@ example, this code will work as expected when using ``py::class_``: m.def("create_example", &create_example); -However, this will fail with ``py::class_`` (but work with +However, this will fail with ``py::class_`` (but works with ``py::classh``): .. code-block:: cpp @@ -102,11 +88,10 @@ It is possible to use ``std::shared_ptr`` as the holder, for example: Compared to using ``py::classh``, there are two noteworthy disadvantages: -* A ``py::class_`` for any particular C++ type ``T`` (and all its derived types) - can only use a single holder type. Therefore, ``std::unique_ptr`` - cannot even be passed from C++ to Python if the ``std::shared_ptr`` holder - is used. This will become apparent only at runtime, often through a - segmentation fault or double free. +* Because a ``py::class_`` for a given C++ type ``T`` can only use a + single holder type, ``std::unique_ptr`` cannot even be passed from C++ + to Python. This will become apparent only at runtime, often through a + segmentation fault. * Similar to the ``std::unique_ptr`` holder, the handling of base-and-derived classes involves a ``reinterpret_cast`` that has strictly speaking undefined @@ -118,9 +103,9 @@ Compared to using ``py::classh``, there are two noteworthy disadvantages: Custom smart pointers ===================== -For custom smart pointer, transparent conversions can be enabled -using a macro invocation similar to the following. It must be declared at the -top namespace level before any binding code: +For custom smart pointers (e.g. ``c10::intrusive_ptr`` in pytorch), transparent +conversions can be enabled using a macro invocation similar to the following. +It must be declared at the top namespace level before any binding code: .. code-block:: cpp @@ -167,6 +152,12 @@ specialized: The above specialization informs pybind11 that the custom ``SmartPtr`` class provides ``.get()`` functionality via ``.getPointer()``. +.. note:: + + The two noteworthy disadvantages mentioned under the ``std::shared_ptr`` + section apply similarly to custom smart pointer holders, but there is no + established safe alternative in this case. + .. seealso:: The file :file:`tests/test_smart_ptr.cpp` contains a complete example @@ -174,12 +165,15 @@ provides ``.get()`` functionality via ``.getPointer()``. in more detail. -Be careful to not undermine automatic lifetime management -========================================================= +Be careful not to accidentally undermine automatic lifetime management +====================================================================== -One potential stumbling block when using holder types is that they need to be -applied consistently. Can you guess what's broken about the following binding -code? +``py::class_``-wrapped objects automatically manage the lifetime of the +wrapped C++ object, in collaboration with the chosen holder type. +When wrapping C++ functions involving raw pointers, care needs to be taken +to not inadvertently transfer ownership, resulting in multiple Python +objects acting as owners, causing heap-use-after-free or double-free errors. +For example: .. code-block:: cpp @@ -188,7 +182,7 @@ code? class Parent { public: Parent() : child(std::make_shared()) { } - Child *get_child() { return child.get(); } /* Hint: ** DON'T DO THIS ** */ + Child *get_child() { return child.get(); } /* DANGER */ private: std::shared_ptr child; }; @@ -198,7 +192,7 @@ code? py::class_>(m, "Parent") .def(py::init<>()) - .def("get_child", &Parent::get_child); + .def("get_child", &Parent::get_child); /* PROBLEM */ } The following Python code will cause undefined behavior (and likely a @@ -210,34 +204,18 @@ segmentation fault). print(Parent().get_child()) -The problem is that ``Parent::get_child()`` returns a pointer to an instance of -``Child``, but the fact that this instance is already managed by -``std::shared_ptr<...>`` is lost when passing raw pointers. In this case, -pybind11 will create a second independent ``std::shared_ptr<...>`` that also -claims ownership of the pointer. In the end, the object will be freed **twice** -since these shared pointers have no way of knowing about each other. - -There are two ways to resolve this issue: - -1. For types that are managed by a smart pointer class, never use raw pointers - in function arguments or return values. In other words: always consistently - wrap pointers into their designated holder types (such as - ``std::shared_ptr<...>``). In this case, the signature of ``get_child()`` - should be modified as follows: - -.. code-block:: cpp - - std::shared_ptr get_child() { return child; } - -2. Adjust the definition of ``Child`` by specifying - ``std::enable_shared_from_this`` (see cppreference_ for details) as a - base class. This adds a small bit of information to ``Child`` that allows - pybind11 to realize that there is already an existing - ``std::shared_ptr<...>`` and communicate with it. In this case, the - declaration of ``Child`` should look as follows: - -.. _cppreference: http://en.cppreference.com/w/cpp/memory/enable_shared_from_this - -.. code-block:: cpp - - class Child : public std::enable_shared_from_this { }; +Part of the ``/* PROBLEM */`` here is that pybind11 falls back to using +``return_value_policy::take_ownership`` as the default (see +:ref:`return_value_policies`). The fact that the ``Child`` instance is +already managed by ``std::shared_ptr`` is lost. Therefore pybind11 +will create a second independent ``std::shared_ptr`` that also +claims ownership of the pointer, eventually leading to heap-use-after-free +or double-free errors. + +There are various ways to resolve this issue, either by changing +the ``Child`` or ``Parent`` C++ implementations (e.g. using +``std::enable_shared_from_this`` as a base class for +``Child``, or adding a member function to ``Parent`` that returns +``std::shared_ptr``), or if that is not feasible, by using +``return_value_policy::reference_internal``. What is the best approach +depends on the exact situation. diff --git a/docs/classes.rst b/docs/classes.rst index 4f2167dac1..4ae47053c0 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -37,8 +37,14 @@ The binding code for ``Pet`` looks as follows: :class:`class_` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding -constructor (see the :ref:`custom_constructors` section for details). An -interactive Python session demonstrating this example is shown below: +constructor (see the :ref:`custom_constructors` section for details). + +.. note:: + + Starting with pybind11v3, it is recommended to use `py::classh` in most + situations. See :ref:`smart_holder` for more information. + +An interactive Python session demonstrating this example is shown below: .. code-block:: pycon From eb550d03d3b23c02bbef38f4633ce3d4475cef92 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 12:43:04 -0800 Subject: [PATCH 06/25] Remove README_smart_holder.rst --- README_smart_holder.rst | 123 ---------------------------------------- 1 file changed, 123 deletions(-) delete mode 100644 README_smart_holder.rst diff --git a/README_smart_holder.rst b/README_smart_holder.rst deleted file mode 100644 index 05b1ab56fa..0000000000 --- a/README_smart_holder.rst +++ /dev/null @@ -1,123 +0,0 @@ -============================== -pybind11 — smart_holder branch -============================== - - -Overview -======== - -- The smart_holder branch is a strict superset of the pybind11 master branch. - Everything that works with the master branch is expected to work exactly the - same with the smart_holder branch. - -- Activating the smart_holder functionality for a given C++ type ``T`` is as - easy as changing ``py::class_`` to ``py::classh`` in client code. - -- The ``py::classh`` functionality includes - - * support for **two-way** Python/C++ conversions for both - ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. - — In contrast, ``py::class_`` only supports one-way C++-to-Python - conversions for ``std::unique_ptr``, or alternatively two-way - Python/C++ conversions for ``std::shared_ptr``, which then excludes - the one-way C++-to-Python ``std::unique_ptr`` conversions (this - manifests itself through undefined runtime behavior). - - * passing a Python object back to C++ via ``std::unique_ptr``, safely - **disowning** the Python object. - - * safely passing `"trampoline" - `_ - objects (objects with C++ virtual function overrides implemented in - Python) via ``std::unique_ptr`` or ``std::shared_ptr`` back to C++: - associated Python objects are automatically kept alive for the lifetime - of the smart-pointer. - -Note: As of `PR #5257 `_ -the smart_holder functionality is fully baked into pybind11. -Prior to PR #5257 the smart_holder implementation was an "add-on", which made -it necessary to use a ``PYBIND11_SMART_HOLDER_TYPE_CASTERS`` macro. This macro -still exists for backward compatibility, but is now a no-op. The trade-off -for this convenience is that the ``PYBIND11_INTERNALS_VERSION`` needed to be -changed. Consequently, Python extension modules built with the smart_holder -branch no longer interoperate with extension modules built with the pybind11 -master branch. If cross-extension-module interoperability is required, all -extension modules involved must be built with the smart_holder branch. -— Probably, most extension modules do not require cross-extension-module -interoperability, but exceptions to this are quite common. - - -What is fundamentally different? --------------------------------- - -- Classic pybind11 has the concept of "smart-pointer is holder". - Interoperability between smart-pointers is completely missing. For example, - with ``py::class_>``, ``return``-ing a - ``std::unique_ptr`` leads to undefined runtime behavior - (`#1138 `_). - A `systematic analysis can be found here - `_. - -- ``py::smart_holder`` has a richer concept in comparison, with well-defined - runtime behavior in all situations. ``py::smart_holder`` "knows" about both - ``std::unique_ptr`` and ``std::shared_ptr``, and how they interoperate. - - -What motivated the development of the smart_holder code? --------------------------------------------------------- - -- The original context was retooling of `PyCLIF - `_, to use pybind11 underneath, - instead of directly targeting the Python C API. Essentially the smart_holder - branch is porting established PyCLIF functionality into pybind11. (However, - this work also led to bug fixes in PyCLIF.) - - -Installation -============ - -Currently ``git clone`` is the only option. We do not have released packages. - -.. code-block:: bash - - git clone --branch smart_holder https://github.com/pybind/pybind11.git - -Everything else is exactly identical to using the default (master) branch. - - -Trampolines and std::unique_ptr -------------------------------- - -A pybind11 `"trampoline" -`_ -is a C++ helper class with virtual function overrides that transparently -call back from C++ into Python. To enable safely passing a ``std::unique_ptr`` -to a trampoline object between Python and C++, the trampoline class must -inherit from ``py::trampoline_self_life_support``, for example: - -.. code-block:: cpp - - class PyAnimal : public Animal, public py::trampoline_self_life_support { - ... - }; - -This is the only difference compared to classic pybind11. A fairly -minimal but complete example is tests/test_class_sh_trampoline_unique_ptr.cpp. - - -Related links -============= - -* The smart_holder branch addresses issue - `#1138 `_ and - the ten issues enumerated in the `description of PR 2839 - `_. - -* `Description of PR #2672 - `_, from which - the smart_holder branch was created. - -* Small `slide deck - `_ - presented in meeting with pybind11 maintainers on Feb 22, 2021. Slides 5 - and 6 show performance comparisons. (These are outdated but probably not far off.) From 860d58e7cc1722566b9b0c12ca8cd6cc3b9e9e16 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 12:44:08 -0800 Subject: [PATCH 07/25] Restore original README.rst from master --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 54da6349f9..8e0c391770 100644 --- a/README.rst +++ b/README.rst @@ -13,10 +13,6 @@ .. start -.. Note:: - - This is the pybind11 **smart_holder** branch. Please refer to - ``README_smart_holder.rst`` for branch-specific information. **pybind11** is a lightweight header-only library that exposes C++ types in Python and vice versa, mainly to create Python bindings of existing From 6bf9e8862f8e6a30c7b67e567c370c0cc1e17d4f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 12:46:25 -0800 Subject: [PATCH 08/25] [ci skip] Minimal change to README.rst, to leave a hint that this is pybind11v3 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8e0c391770..493d111634 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ .. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png :alt: pybind11 logo -**pybind11 — Seamless operability between C++11 and Python** +**pybind11 (v3) — Seamless interoperability between C++ and Python** |Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status| From f4ad02b0e0a63737b398fc50fc8937b025e857cc Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 13:40:54 -0800 Subject: [PATCH 09/25] [ci skip] Work in ChatGPT suggestions. --- docs/advanced/smart_ptrs.rst | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 35ebd82e07..9bb82c5a00 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -20,10 +20,10 @@ a type ``T`` is ``std::unique_ptr``. Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is the recommended ``py::class_`` holder for most situations, but it is **not** the default holder, and there are no plans to make it the default holder in -the future. This is based on the assumption that such a change would cause -more disruption than it is worth, especially because it is extremely easy -to use the safer and more versatile ``py::smart_holder``. For a given C++ -type ``T``, simply change +the future. This decision is based on the assumption that such a change would +cause more disruption than benefit, especially because it is extremely easy +to use the safer and more versatile ``py::smart_holder``. To use +``py::smart_holder`` for a given C++ type ``T``, simply change * ``py::class_`` to @@ -32,6 +32,8 @@ type ``T``, simply change .. note:: ``py::classh`` is a shortcut for ``py::class_``. + — The ``h`` in ``py::classh`` comes from **smart_holder** but is condensed + for brevity. The ``py::classh`` functionality includes the following: @@ -54,12 +56,12 @@ The ``py::classh`` functionality includes the following: ``std::unique_ptr`` =================== -This is the default ``py::class_`` holder and works as expected in most -situations. However, note that the handling of base-and-derived classes -involves a ``reinterpret_cast`` that has strictly speaking undefined -behavior. Also note that the ``std::unique_ptr`` holder only supports passing -a ``std::unique_ptr`` from C++ to Python, but not the other way around. For -example, this code will work as expected when using ``py::class_``: +This is the default ``py::class_`` holder and works as expected in +most situations. However, handling base-and-derived classes involves a +``reinterpret_cast``, which is, strictly speaking, undefined behavior. +Also note that the ``std::unique_ptr`` holder only supports passing a +``std::unique_ptr`` from C++ to Python, but not the other way around. +For example, the following code works as expected with ``py::class_``: .. code-block:: cpp @@ -195,8 +197,8 @@ For example: .def("get_child", &Parent::get_child); /* PROBLEM */ } -The following Python code will cause undefined behavior (and likely a -segmentation fault). +The following Python code leads to undefined behavior, likely resulting in +a segmentation fault. .. code-block:: python From 7cae21faf0daef104f4752a9312b8edd45e96772 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Feb 2025 10:34:53 -0800 Subject: [PATCH 10/25] Change macro name to PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE --- .github/workflows/ci.yml | 8 ++++---- include/pybind11/pybind11.h | 9 ++++----- tests/test_class.cpp | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8607ee988..c5c75146f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,20 +88,20 @@ jobs: # Extra ubuntu latest job - runs-on: ubuntu-latest python: '3.11' - # Exercise PYBIND11_USE_SMART_HOLDER_AS_DEFAULT + # Run tests with py::smart_holder as the default holder # with recent (or ideally latest) released Python version. - runs-on: ubuntu-latest python: '3.12' args: > - -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT" + -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE" - runs-on: macos-13 python: '3.12' args: > - -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT" + -DCMAKE_CXX_FLAGS="-DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE" - runs-on: windows-2022 python: '3.12' args: > - -DCMAKE_CXX_FLAGS="/DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT /GR /EHsc" + -DCMAKE_CXX_FLAGS="/DPYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE /GR /EHsc" exclude: # The setup-python action currently doesn't have graalpy for windows - python: 'graalpy-24.1' diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 5564f040e5..3b06eb548e 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1839,14 +1839,13 @@ struct property_cpp_function< detail::both_t_and_d_use_type_caster_base>::value>> : detail::property_cpp_function_sh_unique_ptr_member {}; -#if defined(PYBIND11_USE_SMART_HOLDER_AS_DEFAULT) -// NOTE: THIS IS MEANT FOR STRESS-TESTING ONLY! -// As of PR #5257, for production use, there is no longer a strong reason to make -// smart_holder the default holder: -// Simply use `py::classh` (see below) instead of `py::class_` as needed. +#ifdef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE +// NOTE: THIS IS MEANT FOR STRESS-TESTING OR TRIAGING ONLY! // Running the pybind11 unit tests with smart_holder as the default holder is to ensure // that `py::smart_holder` / `py::classh` is backward-compatible with all pre-existing // functionality. +// Be careful not to link translation units compiled with different default holders, because +// this will cause ODR violations (https://en.wikipedia.org/wiki/One_Definition_Rule). template using default_holder_type = smart_holder; #else diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 86c8d37017..28dc313334 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -632,7 +632,7 @@ CHECK_NOALIAS(8); CHECK_HOLDER(1, unique); CHECK_HOLDER(2, unique); CHECK_HOLDER(3, unique); -#ifndef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT +#ifndef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE CHECK_HOLDER(4, unique); CHECK_HOLDER(5, unique); #endif From 388fa9997b58fb5562483a519565fa72bc8796c7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Feb 2025 21:40:35 -0800 Subject: [PATCH 11/25] Add a note pointing to the holder reinterpret_cast. --- docs/advanced/smart_ptrs.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 9bb82c5a00..35008dbe8f 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -78,6 +78,13 @@ However, this will fail with ``py::class_`` (but works with void do_something_with_example(std::unique_ptr ex) { ... } +.. note:: + + The ``reinterpret_cast`` mentioned above is `here + `_. + For completeness: The same cast is also applied to ``py::smart_holder``, + but that is safe, because ``py::smart_holder`` is not templated. + ``std::shared_ptr`` =================== From 449ccebc8a64894c45588660e32b5e11ba2885e3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Feb 2025 10:16:22 -0800 Subject: [PATCH 12/25] Incorporate suggestion by @virtuald: https://github.com/pybind/pybind11/pull/5542#discussion_r1967000989 --- docs/advanced/smart_ptrs.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 35008dbe8f..9449d7d626 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -18,12 +18,12 @@ a type ``T`` is ``std::unique_ptr``. ==================== Starting with pybind11v3, ``py::smart_holder`` is built into pybind11. It is -the recommended ``py::class_`` holder for most situations, but it is **not** -the default holder, and there are no plans to make it the default holder in -the future. This decision is based on the assumption that such a change would -cause more disruption than benefit, especially because it is extremely easy -to use the safer and more versatile ``py::smart_holder``. To use -``py::smart_holder`` for a given C++ type ``T``, simply change +the recommended ``py::class_`` holder for most situations. However, for +backward compatibility it is **not** the default holder, and there are no +plans to make it the default holder in the future. + +It is extremely easy to use the safer and more versatile ``py::smart_holder``: +simply change * ``py::class_`` to From ac9d31e13fd15f2a8e3deeccd840e5038c028da8 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Feb 2025 12:10:38 -0800 Subject: [PATCH 13/25] Systematically change most py::class_ to py::classh under docs/ --- docs/advanced/cast/eigen.rst | 4 +- docs/advanced/cast/overview.rst | 14 ++-- docs/advanced/cast/stl.rst | 6 +- docs/advanced/classes.rst | 135 ++++++++++++++------------------ docs/advanced/functions.rst | 22 +++--- docs/advanced/misc.rst | 26 +++--- docs/advanced/pycpp/numpy.rst | 6 +- docs/classes.rst | 40 +++++----- 8 files changed, 116 insertions(+), 137 deletions(-) diff --git a/docs/advanced/cast/eigen.rst b/docs/advanced/cast/eigen.rst index 894ce97f3c..0e6be8a143 100644 --- a/docs/advanced/cast/eigen.rst +++ b/docs/advanced/cast/eigen.rst @@ -102,7 +102,7 @@ example: }; // Later, in binding code: - py::class_(m, "MyClass") + py::classh(m, "MyClass") .def(py::init<>()) .def("copy_matrix", &MyClass::getMatrix) // Makes a copy! .def("get_matrix", &MyClass::getMatrix, py::return_value_policy::reference_internal) @@ -250,7 +250,7 @@ copying to take place: // The associated binding code: using namespace pybind11::literals; // for "arg"_a - py::class_(m, "MyClass") + py::classh(m, "MyClass") // ... other class definitions .def("some_method", &MyClass::some_method, py::arg().noconvert()); diff --git a/docs/advanced/cast/overview.rst b/docs/advanced/cast/overview.rst index d5a34ef942..2594924b7e 100644 --- a/docs/advanced/cast/overview.rst +++ b/docs/advanced/cast/overview.rst @@ -3,13 +3,13 @@ Overview .. rubric:: 1. Native type in C++, wrapper in Python -Exposing a custom C++ type using :class:`py::class_` was covered in detail -in the :doc:`/classes` section. There, the underlying data structure is -always the original C++ class while the :class:`py::class_` wrapper provides -a Python interface. Internally, when an object like this is sent from C++ to -Python, pybind11 will just add the outer wrapper layer over the native C++ -object. Getting it back from Python is just a matter of peeling off the -wrapper. +Exposing a custom C++ type using ``py::classh`` (or ``py::class_``) was +covered in detail in the :doc:`/classes` section. There, the underlying +data structure is always the original C++ class while the ``py::classh`` +wrapper provides a Python interface. Internally, when an object like this +is sent from C++ to Python, pybind11 will just add the outer wrapper layer +over the native C++ object. Getting it back from Python is just a matter of +peeling off the wrapper. .. rubric:: 2. Wrapper in C++, native type in Python diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 42b85532d8..169c17d7d2 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -135,7 +135,7 @@ functions: /* ... binding code ... */ - py::class_(m, "MyClass") + py::classh(m, "MyClass") .def(py::init<>()) .def_readwrite("contents", &MyClass::contents); @@ -169,12 +169,12 @@ macro must be specified at the top level (and outside of any namespaces), since it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must -also have a corresponding ``class_`` declaration to associate them with a name +also have a corresponding ``py::classh`` declaration to associate them with a name in Python, and to define a set of available operations, e.g.: .. code-block:: cpp - py::class_>(m, "IntVector") + py::classh>(m, "IntVector") .def(py::init<>()) .def("clear", &std::vector::clear) .def("pop_back", &std::vector::pop_back) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 45c038ef7a..dbe4a6f39c 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -46,10 +46,10 @@ Normally, the binding code for these classes would look as follows: .. code-block:: cpp PYBIND11_MODULE(example, m) { - py::class_(m, "Animal") + py::classh(m, "Animal") .def("go", &Animal::go); - py::class_(m, "Dog") + py::classh(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -64,7 +64,7 @@ helper class that is defined as follows: .. code-block:: cpp - class PyAnimal : public Animal { + class PyAnimal : public Animal, py::trampoline_self_life_support { public: /* Inherit the constructors */ using Animal::Animal; @@ -80,6 +80,11 @@ helper class that is defined as follows: } }; +The ``py::trampoline_self_life_support`` base class is needed to ensure that a +``std::unique_ptr`` can safely be passed between Python and C++. It is best +practice to always use the base class, although it may not strictly be needed +in all use cases. + The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have a default implementation. There are also two alternate macros @@ -95,28 +100,30 @@ The binding code also needs a few minor adaptations (highlighted): :emphasize-lines: 2,3 PYBIND11_MODULE(example, m) { - py::class_(m, "Animal") + py::classh(m, "Animal") .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog") + py::classh(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); } Importantly, pybind11 is made aware of the trampoline helper class by -specifying it as an extra template argument to :class:`class_`. (This can also -be combined with other template arguments such as a custom holder type; the -order of template types does not matter). Following this, we are able to -define a constructor as usual. +specifying it as an extra template argument to ``py::classh``. (This can also +be combined with other template arguments; the order of template types does +not matter). Using ``py::classh`` (vs ``py::class_``) is to ensure that that +a ``std::unique_ptr`` can safely be passed between Python and C++ (see +:ref:`smart_holder`). -Bindings should be made against the actual class, not the trampoline helper class. +A constructor can be defined as usual. Bindings should be made against the +actual class, not the trampoline helper class. .. code-block:: cpp :emphasize-lines: 3 - py::class_(m, "Animal"); + py::classh(m, "Animal"); .def(py::init<>()) .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ @@ -125,32 +132,6 @@ extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the necessary steps required to providing proper overriding support for inherited classes. -To enable safely passing a ``std::unique_ptr`` to a trampoline object between -Python and C++, - -1. the C++ type (``Animal`` above) must be wrapped with ``py::classh`` - (see :ref:`smart_holder`), and - -2. the trampoline helper class must inherit from - ``py::trampoline_self_life_support``. - -I.e. the example above needs these two changes: - -.. code-block:: cpp - - class PyAnimal : public Animal, public py::trampoline_self_life_support { - ... - }; - -.. code-block:: cpp - - py::classh(m, "Animal"); - -.. seealso:: - - A fairly minimal but complete example is in - :file:`tests/test_class_sh_trampoline_unique_ptr.cpp`. - The Python session below shows how to override ``Animal::go`` and invoke it via a virtual method call. @@ -337,9 +318,9 @@ The classes are then registered with pybind11 using: .. code-block:: cpp - py::class_> animal(m, "Animal"); - py::class_> dog(m, "Dog"); - py::class_> husky(m, "Husky"); + py::classh> animal(m, "Animal"); + py::classh> dog(m, "Dog"); + py::classh> husky(m, "Husky"); // ... add animal, dog, husky definitions Note that ``Husky`` did not require a dedicated trampoline template class at @@ -456,7 +437,7 @@ class like this: static Example create(int a) { return Example(a); } }; - py::class_(m, "Example") + py::classh(m, "Example") .def(py::init(&Example::create)); While it is possible to create a straightforward binding of the static @@ -483,7 +464,7 @@ The following example shows the different approaches: Example(std::string); }; - py::class_(m, "Example") + py::classh(m, "Example") // Bind the factory function as a constructor: .def(py::init(&Example::create)) // Bind a lambda function returning a pointer wrapped in a holder: @@ -530,7 +511,7 @@ an alias: using Example::Example; PyExample(Example &&base) : Example(std::move(base)) {} }; - py::class_(m, "Example") + py::classh(m, "Example") // Returns an Example pointer. If a PyExample is needed, the Example // instance will be moved via the extra constructor in PyExample, above. .def(py::init([]() { return new Example(); })) @@ -555,7 +536,7 @@ constructor of the target class. This means that it can be used to bind std::string b; }; - py::class_(m, "Aggregate") + py::classh(m, "Aggregate") .def(py::init()); .. note:: @@ -572,13 +553,11 @@ Non-public destructors If a class has a private or protected destructor (as might e.g. be the case in a singleton pattern), a compile error will occur when creating bindings via -pybind11. The underlying issue is that the ``std::unique_ptr`` holder type that -is responsible for managing the lifetime of instances will reference the -destructor even if no deallocations ever take place. In order to expose classes -with private or protected destructors, it is possible to override the holder -type via a holder type argument to ``class_``. Pybind11 provides a helper class -``py::nodelete`` that disables any destructor invocations. In this case, it is -crucial that instances are deallocated on the C++ side to avoid memory leaks. +pybind11. In order to expose classes with private or protected destructors, +it is possible to override the holder type via a holder type argument to +``class_``. Pybind11 provides a helper class ``py::nodelete`` that disables +any destructor invocations. (If the instance is not a singleton, it is +crucial that is deallocated on the C++ side to avoid memory leaks.) .. code-block:: cpp @@ -645,10 +624,10 @@ could be a fixed and an arbitrary precision number type). .. code-block:: cpp - py::class_(m, "A") + py::classh(m, "A") /// ... members ... - py::class_(m, "B") + py::classh(m, "B") .def(py::init()) /// ... members ... @@ -695,7 +674,7 @@ that ignores it: .. code-block:: cpp - py::class_(m, "Foo") + py::classh(m, "Foo") .def_property_readonly_static("foo", [](py::object /* self */) { return Foo(); }); Operator overloading @@ -735,7 +714,7 @@ to Python. #include PYBIND11_MODULE(example, m) { - py::class_(m, "Vector2") + py::classh(m, "Vector2") .def(py::init()) .def(py::self + py::self) .def(py::self += py::self) @@ -807,7 +786,7 @@ to bind these two functions: .. code-block:: cpp - py::class_(m, "Pickleable") + py::classh(m, "Pickleable") .def(py::init()) .def("value", &Pickleable::value) .def("extra", &Pickleable::extra) @@ -878,7 +857,7 @@ which should look as follows: .. code-block:: cpp - py::class_(m, "Copyable") + py::classh(m, "Copyable") .def("__copy__", [](const Copyable &self) { return Copyable(self); }) @@ -897,11 +876,11 @@ Multiple Inheritance pybind11 can create bindings for types that derive from multiple base types (aka. *multiple inheritance*). To do so, specify all bases in the template -arguments of the ``class_`` declaration: +arguments of the ``py::classh`` declaration: .. code-block:: cpp - py::class_(m, "MyType") + py::classh(m, "MyType") ... The base types can be specified in arbitrary order, and they can even be @@ -921,7 +900,7 @@ inheritance, which can lead to undefined behavior. In such cases, add the tag .. code-block:: cpp - py::class_(m, "MyType", py::multiple_inheritance()); + py::classh(m, "MyType", py::multiple_inheritance()); The tag is redundant and does not need to be specified when multiple base types are listed. @@ -939,7 +918,7 @@ example, this allows the following: .. code-block:: cpp // In the module1.cpp binding code for module1: - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def_readonly("name", &Pet::name); @@ -972,11 +951,11 @@ because of conflicting definitions on the external type: // dogs.cpp // Binding for external library class: - py::class(m, "Pet") + py::classh(m, "Pet") .def("name", &pets::Pet::name); // Binding for local extension class: - py::class(m, "Dog") + py::classh(m, "Dog") .def(py::init()); .. code-block:: cpp @@ -984,11 +963,11 @@ because of conflicting definitions on the external type: // cats.cpp, in a completely separate project from the above dogs.cpp. // Binding for external library class: - py::class(m, "Pet") + py::classh(m, "Pet") .def("get_name", &pets::Pet::name); // Binding for local extending class: - py::class(m, "Cat") + py::classh(m, "Cat") .def(py::init()); .. code-block:: pycon @@ -1001,18 +980,18 @@ because of conflicting definitions on the external type: To get around this, you can tell pybind11 to keep the external class binding localized to the module by passing the ``py::module_local()`` attribute into -the ``py::class_`` constructor: +the ``py::classh`` constructor: .. code-block:: cpp // Pet binding in dogs.cpp: - py::class(m, "Pet", py::module_local()) + py::classh(m, "Pet", py::module_local()) .def("name", &pets::Pet::name); .. code-block:: cpp // Pet binding in cats.cpp: - py::class(m, "Pet", py::module_local()) + py::classh(m, "Pet", py::module_local()) .def("get_name", &pets::Pet::name); This makes the Python-side ``dogs.Pet`` and ``cats.Pet`` into distinct classes, @@ -1087,7 +1066,7 @@ It's normally not possible to expose ``protected`` member functions to Python: int foo() const { return 42; } }; - py::class_(m, "A") + py::classh(m, "A") .def("foo", &A::foo); // error: 'foo' is a protected member of 'A' On one hand, this is good because non-``public`` members aren't meant to be @@ -1108,7 +1087,7 @@ The following pattern makes this possible: using A::foo; // inherited with different access modifier }; - py::class_(m, "A") // bind the primary class + py::classh(m, "A") // bind the primary class .def("foo", &Publicist::foo); // expose protected methods via the publicist This works because ``&Publicist::foo`` is exactly the same function as @@ -1140,7 +1119,7 @@ described trampoline: using A::foo; }; - py::class_(m, "A") // <-- `Trampoline` here + py::classh(m, "A") // <-- `Trampoline` here .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`! Binding final classes @@ -1156,7 +1135,7 @@ to be declared final. class IsFinal final {}; - py::class_(m, "IsFinal", py::is_final()); + py::classh(m, "IsFinal", py::is_final()); When you try to inherit from such a class in Python, you will now get this error: @@ -1194,7 +1173,7 @@ wrap instantiated templated classes. You cannot wrap a non-instantiated template .. code-block:: cpp // BROKEN (this will not compile) - py::class_(m, "Cage"); + py::classh(m, "Cage"); .def("get", &Cage::get); You must explicitly specify each template/type combination that you want to @@ -1203,11 +1182,11 @@ wrap separately. .. code-block:: cpp // ok - py::class_>(m, "CatCage") + py::classh>(m, "CatCage") .def("get", &Cage::get); // ok - py::class_>(m, "DogCage") + py::classh>(m, "DogCage") .def("get", &Cage::get); If your class methods have template parameters you can wrap those as well, @@ -1221,7 +1200,7 @@ but once again each instantiation must be explicitly specified: T fn(V v); }; - py::class>(m, "MyClassT") + py::classh>(m, "MyClassT") .def("fn", &MyClass::fn); Custom automatic downcasters @@ -1330,7 +1309,7 @@ Custom type setup For advanced use cases, such as enabling garbage collection support, you may wish to directly manipulate the ``PyHeapTypeObject`` corresponding to a -``py::class_`` definition. +``py::classh`` definition. You can do that using ``py::custom_type_setup``: @@ -1339,7 +1318,7 @@ You can do that using ``py::custom_type_setup``: struct OwnsPythonObjects { py::object value = py::none(); }; - py::class_ cls( + py::classh cls( m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index ff00c9c8ac..4313c1b41d 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -110,7 +110,7 @@ Return value policies can also be applied to properties: .. code-block:: cpp - class_(m, "MyClass") + py::classh(m, "MyClass") .def_property("data", &MyClass::getData, &MyClass::setData, py::return_value_policy::copy); @@ -121,7 +121,7 @@ targeted arguments can be passed through the :class:`cpp_function` constructor: .. code-block:: cpp - class_(m, "MyClass") + py::classh(m, "MyClass") .def_property("data", py::cpp_function(&MyClass::getData, py::return_value_policy::copy), py::cpp_function(&MyClass::setData) @@ -194,7 +194,7 @@ container: .. code-block:: cpp - py::class_(m, "List") + py::classh(m, "List") .def("append", &List::append, py::keep_alive<1, 2>()); For consistency, the argument indexing is identical for constructors. Index @@ -205,7 +205,7 @@ ties the lifetime of the constructor element to the constructed object: .. code-block:: cpp - py::class_(m, "Nurse") + py::classh(m, "Nurse") .def(py::init(), py::keep_alive<1, 2>()); .. note:: @@ -332,11 +332,11 @@ Consider the following example: .. code-block:: cpp - py::class_("MyClass") + py::classh("MyClass") .def("myFunction", py::arg("arg") = SomeType(123)); In this case, pybind11 must already be set up to deal with values of the type -``SomeType`` (via a prior instantiation of ``py::class_``), or an +``SomeType`` (via a prior instantiation of ``py::classh``), or an exception will be thrown. Another aspect worth highlighting is that the "preview" of the default argument @@ -357,7 +357,7 @@ default argument manually using the ``arg_v`` notation: .. code-block:: cpp - py::class_("MyClass") + py::classh("MyClass") .def("myFunction", py::arg_v("arg", SomeType(123), "SomeType(123)")); Sometimes it may be necessary to pass a null pointer value as a default @@ -366,7 +366,7 @@ like so: .. code-block:: cpp - py::class_("MyClass") + py::classh("MyClass") .def("myFunction", py::arg("arg") = static_cast(nullptr)); .. _keyword_only_arguments: @@ -489,7 +489,7 @@ name, i.e. by specifying ``py::arg().noconvert()``. Allow/Prohibiting None arguments ================================ -When a C++ type registered with :class:`py::class_` is passed as an argument to +When a C++ type registered with :class:`py::classh` is passed as an argument to a function taking the instance as pointer or shared holder (e.g. ``shared_ptr`` or a custom, copyable holder as described in :ref:`smart_pointers`), pybind allows ``None`` to be passed from Python which results in calling the C++ @@ -500,8 +500,8 @@ To explicitly enable or disable this behaviour, using the .. code-block:: cpp - py::class_(m, "Dog").def(py::init<>()); - py::class_(m, "Cat").def(py::init<>()); + py::classh(m, "Dog").def(py::init<>()); + py::classh(m, "Cat").def(py::init<>()); m.def("bark", [](Dog *dog) -> std::string { if (dog) return "woof!"; /* Called with a Dog instance */ else return "(no dog)"; /* Called with None, dog == nullptr */ diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index a256da54a9..0ba3108690 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -80,7 +80,7 @@ could be realized as follows (important changes highlighted): .. code-block:: cpp :emphasize-lines: 8,30,31 - class PyAnimal : public Animal { + class PyAnimal : public Animal, py::trampoline_self_life_support { public: /* Inherit the constructors */ using Animal::Animal; @@ -98,12 +98,12 @@ could be realized as follows (important changes highlighted): }; PYBIND11_MODULE(example, m) { - py::class_ animal(m, "Animal"); + py::classh animal(m, "Animal"); animal .def(py::init<>()) .def("go", &Animal::go); - py::class_(m, "Dog", animal) + py::classh(m, "Dog", animal) .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { @@ -177,30 +177,30 @@ from Section :ref:`inheritance`. .. code-block:: cpp - py::class_ pet(m, "Pet"); + py::classh pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name); - py::class_(m, "Dog", pet /* <- specify parent */) + py::classh(m, "Dog", pet /* <- specify parent */) .def(py::init()) .def("bark", &Dog::bark); Suppose now that ``Pet`` bindings are defined in a module named ``basic``, whereas the ``Dog`` bindings are defined somewhere else. The challenge is of course that the variable ``pet`` is not available anymore though it is needed -to indicate the inheritance relationship to the constructor of ``class_``. -However, it can be acquired as follows: +to indicate the inheritance relationship to the constructor of +``py::classh``. However, it can be acquired as follows: .. code-block:: cpp py::object pet = (py::object) py::module_::import("basic").attr("Pet"); - py::class_(m, "Dog", pet) + py::classh(m, "Dog", pet) .def(py::init()) .def("bark", &Dog::bark); Alternatively, you can specify the base class as a template parameter option to -``class_``, which performs an automated lookup of the corresponding Python +``py::classh``, which performs an automated lookup of the corresponding Python type. Like the above code, however, this also requires invoking the ``import`` function once to ensure that the pybind11 binding code of the module ``basic`` has been executed: @@ -209,7 +209,7 @@ has been executed: py::module_::import("basic"); - py::class_(m, "Dog") + py::classh(m, "Dog") .def(py::init()) .def("bark", &Dog::bark); @@ -382,7 +382,7 @@ Avoiding C++ types in docstrings Docstrings are generated at the time of the declaration, e.g. when ``.def(...)`` is called. At this point parameter and return types should be known to pybind11. -If a custom type is not exposed yet through a ``py::class_`` constructor or a custom type caster, +If a custom type is not exposed yet through a ``py::classh`` constructor or a custom type caster, its C++ type name will be used instead to generate the signature in the docstring: .. code-block:: text @@ -399,8 +399,8 @@ before they are used as a parameter or return type of a function: PYBIND11_MODULE(example, m) { - auto pyFoo = py::class_(m, "Foo"); - auto pyBar = py::class_(m, "Bar"); + auto pyFoo = py::classh(m, "Foo"); + auto pyBar = py::classh(m, "Bar"); pyFoo.def(py::init()); pyBar.def(py::init()); diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index d09a2cea2c..f734b0bdee 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -33,7 +33,7 @@ completely avoid copy operations with Python expressions like .. code-block:: cpp - py::class_(m, "Matrix", py::buffer_protocol()) + py::classh(m, "Matrix", py::buffer_protocol()) .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( m.data(), /* Pointer to buffer */ @@ -47,7 +47,7 @@ completely avoid copy operations with Python expressions like }); Supporting the buffer protocol in a new type involves specifying the special -``py::buffer_protocol()`` tag in the ``py::class_`` constructor and calling the +``py::buffer_protocol()`` tag in the ``py::classh`` constructor and calling the ``def_buffer()`` method with a lambda function that creates a ``py::buffer_info`` description record on demand describing a given matrix instance. The contents of ``py::buffer_info`` mirror the Python buffer protocol @@ -80,7 +80,7 @@ buffer objects (e.g. a NumPy matrix). typedef Matrix::Scalar Scalar; constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; - py::class_(m, "Matrix", py::buffer_protocol()) + py::classh(m, "Matrix", py::buffer_protocol()) .def(py::init([](py::buffer b) { typedef Eigen::Stride Strides; diff --git a/docs/classes.rst b/docs/classes.rst index 4ae47053c0..6741d73a57 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -28,13 +28,13 @@ The binding code for ``Pet`` looks as follows: namespace py = pybind11; PYBIND11_MODULE(example, m) { - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def("setName", &Pet::setName) .def("getName", &Pet::getName); } -:class:`class_` creates bindings for a C++ *class* or *struct*-style data +``py::classh`` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding constructor (see the :ref:`custom_constructors` section for details). @@ -98,7 +98,7 @@ Lambda function instead: .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def("setName", &Pet::setName) .def("getName", &Pet::getName) @@ -129,7 +129,7 @@ method also exists for ``const`` fields. .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def_readwrite("name", &Pet::name) // ... remainder ... @@ -166,7 +166,7 @@ the setter and getter functions: .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def_property("name", &Pet::getName, &Pet::setName) // ... remainder ... @@ -202,7 +202,7 @@ or :func:`class_::def_property`. .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init<>()) .def_readwrite("name", &Pet::name); @@ -220,7 +220,7 @@ must be added to the :class:`py::class_` constructor: .. code-block:: cpp - py::class_(m, "Pet", py::dynamic_attr()) + py::classh(m, "Pet", py::dynamic_attr()) .def(py::init<>()) .def_readwrite("name", &Pet::name); @@ -268,12 +268,12 @@ parameter of the :class:`class_`: .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def_readwrite("name", &Pet::name); // Method 1: template parameter: - py::class_(m, "Dog") + py::classh(m, "Dog") .def(py::init()) .def("bark", &Dog::bark); @@ -282,12 +282,12 @@ Alternatively, we can also assign a name to the previously bound ``Pet`` .. code-block:: cpp - py::class_ pet(m, "Pet"); + py::classh pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name); - // Method 2: pass parent class_ object: - py::class_(m, "Dog", pet /* <- specify Python parent type */) + // Method 2: pass parent py::classh object: + py::classh(m, "Dog", pet /* <- specify Python parent type */) .def(py::init()) .def("bark", &Dog::bark); @@ -334,8 +334,8 @@ will automatically recognize this: }; // Same binding code - py::class_(m, "PolymorphicPet"); - py::class_(m, "PolymorphicDog") + py::classh(m, "PolymorphicPet"); + py::classh(m, "PolymorphicDog") .def(py::init<>()) .def("bark", &PolymorphicDog::bark); @@ -387,7 +387,7 @@ sequence. .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def(py::init()) .def("set", static_cast(&Pet::set), "Set the pet's age") .def("set", static_cast(&Pet::set), "Set the pet's name"); @@ -418,7 +418,7 @@ syntax to cast the overloaded function: .. code-block:: cpp - py::class_(m, "Pet") + py::classh(m, "Pet") .def("set", py::overload_cast(&Pet::set), "Set the pet's age") .def("set", py::overload_cast(&Pet::set), "Set the pet's name"); @@ -434,7 +434,7 @@ on constness, the ``py::const_`` tag should be used: int foo(int x, float y) const; }; - py::class_(m, "Widget") + py::classh(m, "Widget") .def("foo_mutable", py::overload_cast(&Widget::foo)) .def("foo_const", py::overload_cast(&Widget::foo, py::const_)); @@ -446,7 +446,7 @@ you can use ``py::detail::overload_cast_impl`` with an additional set of parenth template using overload_cast_ = pybind11::detail::overload_cast_impl; - py::class_(m, "Pet") + py::classh(m, "Pet") .def("set", overload_cast_()(&Pet::set), "Set the pet's age") .def("set", overload_cast_()(&Pet::set), "Set the pet's name"); @@ -486,7 +486,7 @@ The binding code for this example looks as follows: .. code-block:: cpp - py::class_ pet(m, "Pet"); + py::classh pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name) @@ -498,7 +498,7 @@ The binding code for this example looks as follows: .value("Cat", Pet::Kind::Cat) .export_values(); - py::class_(pet, "Attributes") + py::classh(pet, "Attributes") .def(py::init<>()) .def_readwrite("age", &Pet::Attributes::age); From a4cfd386e36779c59cd2e598ffae1cb9eeed59a2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Feb 2025 12:21:02 -0800 Subject: [PATCH 14/25] Remove references to README_smart_holder.rst This should have been part of commit eb550d03d3b23c02bbef38f4633ce3d4475cef92. --- MANIFEST.in | 1 - tests/extra_python_package/test_files.py | 1 - 2 files changed, 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index c13394558f..7ce83c5527 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ prune tests -include README_smart_holder.rst recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 4cd4bedfd8..b01b49867f 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -131,7 +131,6 @@ "LICENSE", "MANIFEST.in", "README.rst", - "README_smart_holder.rst", "PKG-INFO", "SECURITY.md", } From 046883cc6a3e316cd4735c0af5c9f4739d9c6a39 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 24 Feb 2025 19:57:00 -0800 Subject: [PATCH 15/25] [ci skip] Fix minor oversight (``class_`` -> ``py::class_``) noticed by chance. --- docs/advanced/classes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index dbe4a6f39c..a736892c96 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -555,7 +555,7 @@ If a class has a private or protected destructor (as might e.g. be the case in a singleton pattern), a compile error will occur when creating bindings via pybind11. In order to expose classes with private or protected destructors, it is possible to override the holder type via a holder type argument to -``class_``. Pybind11 provides a helper class ``py::nodelete`` that disables +``py::class_``. Pybind11 provides a helper class ``py::nodelete`` that disables any destructor invocations. (If the instance is not a singleton, it is crucial that is deallocated on the C++ side to avoid memory leaks.) From 4cc528d15a794cae129c01e2fcb026974484fe00 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Feb 2025 22:23:29 -0800 Subject: [PATCH 16/25] [ci skip] Resolve suggestion by @virtuald https://github.com/pybind/pybind11/pull/5542#discussion_r1969940605 --- docs/advanced/classes.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index a736892c96..450da05ab6 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -80,10 +80,15 @@ helper class that is defined as follows: } }; -The ``py::trampoline_self_life_support`` base class is needed to ensure that a -``std::unique_ptr`` can safely be passed between Python and C++. It is best -practice to always use the base class, although it may not strictly be needed -in all use cases. +The ``py::trampoline_self_life_support`` base class is needed to ensure +that a ``std::unique_ptr`` can safely be passed between Python and C++. To +steer clear of notorious pitfalls (e.g. inheritance slicing), it is best +practice to always use the base class, in combination with ``py::classh``. + +.. note:: + For completeness, the base class has no effect if a holder other than + ``py::smart_holder`` (usually via ``py::classh``) is used. Please think + twice, though, the pitfalls are very real. The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have From 5f1d64639fb6c8eeeaccdd88245670c3db0b6a3d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 26 Feb 2025 09:12:42 -0800 Subject: [PATCH 17/25] [ci skip] Apply suggestions by @timohl (thanks!) * https://github.com/pybind/pybind11/pull/5542#discussion_r1970714551 * https://github.com/pybind/pybind11/pull/5542#discussion_r1971315329 * https://github.com/pybind/pybind11/pull/5542#discussion_r1971322821 --- docs/advanced/classes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 450da05ab6..442c49601f 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -118,19 +118,19 @@ The binding code also needs a few minor adaptations (highlighted): Importantly, pybind11 is made aware of the trampoline helper class by specifying it as an extra template argument to ``py::classh``. (This can also be combined with other template arguments; the order of template types does -not matter). Using ``py::classh`` (vs ``py::class_``) is to ensure that that +not matter). Using ``py::classh`` (vs ``py::class_``) is to ensure that a ``std::unique_ptr`` can safely be passed between Python and C++ (see :ref:`smart_holder`). A constructor can be defined as usual. Bindings should be made against the -actual class, not the trampoline helper class. +actual class, not the trampoline helper class: .. code-block:: cpp :emphasize-lines: 3 py::classh(m, "Animal"); .def(py::init<>()) - .def("go", &PyAnimal::go); /* <--- THIS IS WRONG, use &Animal::go */ + .def("go", &Animal::go); /* <--- DO NOT USE &PyAnimal::go HERE */ Note, however, that the above is sufficient for allowing python classes to extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the @@ -562,7 +562,7 @@ pybind11. In order to expose classes with private or protected destructors, it is possible to override the holder type via a holder type argument to ``py::class_``. Pybind11 provides a helper class ``py::nodelete`` that disables any destructor invocations. (If the instance is not a singleton, it is -crucial that is deallocated on the C++ side to avoid memory leaks.) +crucial that it is deallocated on the C++ side to avoid memory leaks.) .. code-block:: cpp From ff7c0873212be69f1203564363f1a32ef4a10cdf Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 27 Feb 2025 14:12:05 -0800 Subject: [PATCH 18/25] Replace `classh : class_` inhertance with `using`, as suggested by @henryiii https://github.com/pybind/pybind11/pull/5542#issuecomment-2689034104 --- include/pybind11/pybind11.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3b06eb548e..4dee2c55fa 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2378,10 +2378,7 @@ class class_ : public detail::generic_type { // Supports easier switching between py::class_ and py::class_: // users can simply replace the `_` in `class_` with `h` or vice versa. template -class classh : public class_ { -public: - using class_::class_; -}; +using classh = class_; /// Binds an existing constructor taking arguments Args... template From 1e646c91b4cfd78228f7e3f6d2e0d649bad8b30a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Feb 2025 16:55:37 -0500 Subject: [PATCH 19/25] Revert "Systematically change most py::class_ to py::classh under docs/" This reverts commit ac9d31e13fd15f2a8e3deeccd840e5038c028da8. --- docs/advanced/cast/eigen.rst | 4 +- docs/advanced/cast/overview.rst | 14 ++-- docs/advanced/cast/stl.rst | 6 +- docs/advanced/classes.rst | 137 +++++++++++++++++++------------- docs/advanced/functions.rst | 22 ++--- docs/advanced/misc.rst | 26 +++--- docs/advanced/pycpp/numpy.rst | 6 +- docs/classes.rst | 40 +++++----- 8 files changed, 141 insertions(+), 114 deletions(-) diff --git a/docs/advanced/cast/eigen.rst b/docs/advanced/cast/eigen.rst index 0e6be8a143..894ce97f3c 100644 --- a/docs/advanced/cast/eigen.rst +++ b/docs/advanced/cast/eigen.rst @@ -102,7 +102,7 @@ example: }; // Later, in binding code: - py::classh(m, "MyClass") + py::class_(m, "MyClass") .def(py::init<>()) .def("copy_matrix", &MyClass::getMatrix) // Makes a copy! .def("get_matrix", &MyClass::getMatrix, py::return_value_policy::reference_internal) @@ -250,7 +250,7 @@ copying to take place: // The associated binding code: using namespace pybind11::literals; // for "arg"_a - py::classh(m, "MyClass") + py::class_(m, "MyClass") // ... other class definitions .def("some_method", &MyClass::some_method, py::arg().noconvert()); diff --git a/docs/advanced/cast/overview.rst b/docs/advanced/cast/overview.rst index 2594924b7e..d5a34ef942 100644 --- a/docs/advanced/cast/overview.rst +++ b/docs/advanced/cast/overview.rst @@ -3,13 +3,13 @@ Overview .. rubric:: 1. Native type in C++, wrapper in Python -Exposing a custom C++ type using ``py::classh`` (or ``py::class_``) was -covered in detail in the :doc:`/classes` section. There, the underlying -data structure is always the original C++ class while the ``py::classh`` -wrapper provides a Python interface. Internally, when an object like this -is sent from C++ to Python, pybind11 will just add the outer wrapper layer -over the native C++ object. Getting it back from Python is just a matter of -peeling off the wrapper. +Exposing a custom C++ type using :class:`py::class_` was covered in detail +in the :doc:`/classes` section. There, the underlying data structure is +always the original C++ class while the :class:`py::class_` wrapper provides +a Python interface. Internally, when an object like this is sent from C++ to +Python, pybind11 will just add the outer wrapper layer over the native C++ +object. Getting it back from Python is just a matter of peeling off the +wrapper. .. rubric:: 2. Wrapper in C++, native type in Python diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 169c17d7d2..42b85532d8 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -135,7 +135,7 @@ functions: /* ... binding code ... */ - py::classh(m, "MyClass") + py::class_(m, "MyClass") .def(py::init<>()) .def_readwrite("contents", &MyClass::contents); @@ -169,12 +169,12 @@ macro must be specified at the top level (and outside of any namespaces), since it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must -also have a corresponding ``py::classh`` declaration to associate them with a name +also have a corresponding ``class_`` declaration to associate them with a name in Python, and to define a set of available operations, e.g.: .. code-block:: cpp - py::classh>(m, "IntVector") + py::class_>(m, "IntVector") .def(py::init<>()) .def("clear", &std::vector::clear) .def("pop_back", &std::vector::pop_back) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 442c49601f..9fbcc96d5b 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -46,10 +46,10 @@ Normally, the binding code for these classes would look as follows: .. code-block:: cpp PYBIND11_MODULE(example, m) { - py::classh(m, "Animal") + py::class_(m, "Animal") .def("go", &Animal::go); - py::classh(m, "Dog") + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); @@ -64,7 +64,7 @@ helper class that is defined as follows: .. code-block:: cpp - class PyAnimal : public Animal, py::trampoline_self_life_support { + class PyAnimal : public Animal { public: /* Inherit the constructors */ using Animal::Animal; @@ -83,12 +83,13 @@ helper class that is defined as follows: The ``py::trampoline_self_life_support`` base class is needed to ensure that a ``std::unique_ptr`` can safely be passed between Python and C++. To steer clear of notorious pitfalls (e.g. inheritance slicing), it is best -practice to always use the base class, in combination with ``py::classh``. +practice to always use the base class, in combination with +``py::smart_holder``. .. note:: For completeness, the base class has no effect if a holder other than - ``py::smart_holder`` (usually via ``py::classh``) is used. Please think - twice, though, the pitfalls are very real. + ``py::smart_holder`` used, including the default ``std::unique_ptr``. + Please think twice, though, the pitfalls are very real. The macro :c:macro:`PYBIND11_OVERRIDE_PURE` should be used for pure virtual functions, and :c:macro:`PYBIND11_OVERRIDE` should be used for functions which have @@ -105,30 +106,28 @@ The binding code also needs a few minor adaptations (highlighted): :emphasize-lines: 2,3 PYBIND11_MODULE(example, m) { - py::classh(m, "Animal") + py::class_(m, "Animal") .def(py::init<>()) .def("go", &Animal::go); - py::classh(m, "Dog") + py::class_(m, "Dog") .def(py::init<>()); m.def("call_go", &call_go); } Importantly, pybind11 is made aware of the trampoline helper class by -specifying it as an extra template argument to ``py::classh``. (This can also -be combined with other template arguments; the order of template types does -not matter). Using ``py::classh`` (vs ``py::class_``) is to ensure that -a ``std::unique_ptr`` can safely be passed between Python and C++ (see -:ref:`smart_holder`). +specifying it as an extra template argument to :class:`class_`. (This can also +be combined with other template arguments such as a custom holder type; the +order of template types does not matter). Following this, we are able to +define a constructor as usual. -A constructor can be defined as usual. Bindings should be made against the -actual class, not the trampoline helper class: +Bindings should be made against the actual class, not the trampoline helper class. .. code-block:: cpp :emphasize-lines: 3 - py::classh(m, "Animal"); + py::class_(m, "Animal"); .def(py::init<>()) .def("go", &Animal::go); /* <--- DO NOT USE &PyAnimal::go HERE */ @@ -137,6 +136,32 @@ extend ``Animal``, but not ``Dog``: see :ref:`virtual_and_inheritance` for the necessary steps required to providing proper overriding support for inherited classes. +To enable safely passing a ``std::unique_ptr`` to a trampoline object between +Python and C++, + +1. the C++ type (``Animal`` above) must be wrapped with ``py::classh`` + (see :ref:`smart_holder`), and + +2. the trampoline helper class must inherit from + ``py::trampoline_self_life_support``. + +I.e. the example above needs these two changes: + +.. code-block:: cpp + + class PyAnimal : public Animal, public py::trampoline_self_life_support { + ... + }; + +.. code-block:: cpp + + py::classh(m, "Animal"); + +.. seealso:: + + A fairly minimal but complete example is in + :file:`tests/test_class_sh_trampoline_unique_ptr.cpp`. + The Python session below shows how to override ``Animal::go`` and invoke it via a virtual method call. @@ -323,9 +348,9 @@ The classes are then registered with pybind11 using: .. code-block:: cpp - py::classh> animal(m, "Animal"); - py::classh> dog(m, "Dog"); - py::classh> husky(m, "Husky"); + py::class_> animal(m, "Animal"); + py::class_> dog(m, "Dog"); + py::class_> husky(m, "Husky"); // ... add animal, dog, husky definitions Note that ``Husky`` did not require a dedicated trampoline template class at @@ -442,7 +467,7 @@ class like this: static Example create(int a) { return Example(a); } }; - py::classh(m, "Example") + py::class_(m, "Example") .def(py::init(&Example::create)); While it is possible to create a straightforward binding of the static @@ -469,7 +494,7 @@ The following example shows the different approaches: Example(std::string); }; - py::classh(m, "Example") + py::class_(m, "Example") // Bind the factory function as a constructor: .def(py::init(&Example::create)) // Bind a lambda function returning a pointer wrapped in a holder: @@ -516,7 +541,7 @@ an alias: using Example::Example; PyExample(Example &&base) : Example(std::move(base)) {} }; - py::classh(m, "Example") + py::class_(m, "Example") // Returns an Example pointer. If a PyExample is needed, the Example // instance will be moved via the extra constructor in PyExample, above. .def(py::init([]() { return new Example(); })) @@ -541,7 +566,7 @@ constructor of the target class. This means that it can be used to bind std::string b; }; - py::classh(m, "Aggregate") + py::class_(m, "Aggregate") .def(py::init()); .. note:: @@ -558,11 +583,13 @@ Non-public destructors If a class has a private or protected destructor (as might e.g. be the case in a singleton pattern), a compile error will occur when creating bindings via -pybind11. In order to expose classes with private or protected destructors, -it is possible to override the holder type via a holder type argument to -``py::class_``. Pybind11 provides a helper class ``py::nodelete`` that disables -any destructor invocations. (If the instance is not a singleton, it is -crucial that it is deallocated on the C++ side to avoid memory leaks.) +pybind11. The underlying issue is that the ``std::unique_ptr`` holder type that +is responsible for managing the lifetime of instances will reference the +destructor even if no deallocations ever take place. In order to expose classes +with private or protected destructors, it is possible to override the holder +type via a holder type argument to ``class_``. Pybind11 provides a helper class +``py::nodelete`` that disables any destructor invocations. In this case, it is +crucial that instances are deallocated on the C++ side to avoid memory leaks. .. code-block:: cpp @@ -629,10 +656,10 @@ could be a fixed and an arbitrary precision number type). .. code-block:: cpp - py::classh(m, "A") + py::class_(m, "A") /// ... members ... - py::classh(m, "B") + py::class_(m, "B") .def(py::init()) /// ... members ... @@ -679,7 +706,7 @@ that ignores it: .. code-block:: cpp - py::classh(m, "Foo") + py::class_(m, "Foo") .def_property_readonly_static("foo", [](py::object /* self */) { return Foo(); }); Operator overloading @@ -719,7 +746,7 @@ to Python. #include PYBIND11_MODULE(example, m) { - py::classh(m, "Vector2") + py::class_(m, "Vector2") .def(py::init()) .def(py::self + py::self) .def(py::self += py::self) @@ -791,7 +818,7 @@ to bind these two functions: .. code-block:: cpp - py::classh(m, "Pickleable") + py::class_(m, "Pickleable") .def(py::init()) .def("value", &Pickleable::value) .def("extra", &Pickleable::extra) @@ -862,7 +889,7 @@ which should look as follows: .. code-block:: cpp - py::classh(m, "Copyable") + py::class_(m, "Copyable") .def("__copy__", [](const Copyable &self) { return Copyable(self); }) @@ -881,11 +908,11 @@ Multiple Inheritance pybind11 can create bindings for types that derive from multiple base types (aka. *multiple inheritance*). To do so, specify all bases in the template -arguments of the ``py::classh`` declaration: +arguments of the ``class_`` declaration: .. code-block:: cpp - py::classh(m, "MyType") + py::class_(m, "MyType") ... The base types can be specified in arbitrary order, and they can even be @@ -905,7 +932,7 @@ inheritance, which can lead to undefined behavior. In such cases, add the tag .. code-block:: cpp - py::classh(m, "MyType", py::multiple_inheritance()); + py::class_(m, "MyType", py::multiple_inheritance()); The tag is redundant and does not need to be specified when multiple base types are listed. @@ -923,7 +950,7 @@ example, this allows the following: .. code-block:: cpp // In the module1.cpp binding code for module1: - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def_readonly("name", &Pet::name); @@ -956,11 +983,11 @@ because of conflicting definitions on the external type: // dogs.cpp // Binding for external library class: - py::classh(m, "Pet") + py::class(m, "Pet") .def("name", &pets::Pet::name); // Binding for local extension class: - py::classh(m, "Dog") + py::class(m, "Dog") .def(py::init()); .. code-block:: cpp @@ -968,11 +995,11 @@ because of conflicting definitions on the external type: // cats.cpp, in a completely separate project from the above dogs.cpp. // Binding for external library class: - py::classh(m, "Pet") + py::class(m, "Pet") .def("get_name", &pets::Pet::name); // Binding for local extending class: - py::classh(m, "Cat") + py::class(m, "Cat") .def(py::init()); .. code-block:: pycon @@ -985,18 +1012,18 @@ because of conflicting definitions on the external type: To get around this, you can tell pybind11 to keep the external class binding localized to the module by passing the ``py::module_local()`` attribute into -the ``py::classh`` constructor: +the ``py::class_`` constructor: .. code-block:: cpp // Pet binding in dogs.cpp: - py::classh(m, "Pet", py::module_local()) + py::class(m, "Pet", py::module_local()) .def("name", &pets::Pet::name); .. code-block:: cpp // Pet binding in cats.cpp: - py::classh(m, "Pet", py::module_local()) + py::class(m, "Pet", py::module_local()) .def("get_name", &pets::Pet::name); This makes the Python-side ``dogs.Pet`` and ``cats.Pet`` into distinct classes, @@ -1071,7 +1098,7 @@ It's normally not possible to expose ``protected`` member functions to Python: int foo() const { return 42; } }; - py::classh(m, "A") + py::class_(m, "A") .def("foo", &A::foo); // error: 'foo' is a protected member of 'A' On one hand, this is good because non-``public`` members aren't meant to be @@ -1092,7 +1119,7 @@ The following pattern makes this possible: using A::foo; // inherited with different access modifier }; - py::classh(m, "A") // bind the primary class + py::class_(m, "A") // bind the primary class .def("foo", &Publicist::foo); // expose protected methods via the publicist This works because ``&Publicist::foo`` is exactly the same function as @@ -1124,7 +1151,7 @@ described trampoline: using A::foo; }; - py::classh(m, "A") // <-- `Trampoline` here + py::class_(m, "A") // <-- `Trampoline` here .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`! Binding final classes @@ -1140,7 +1167,7 @@ to be declared final. class IsFinal final {}; - py::classh(m, "IsFinal", py::is_final()); + py::class_(m, "IsFinal", py::is_final()); When you try to inherit from such a class in Python, you will now get this error: @@ -1178,7 +1205,7 @@ wrap instantiated templated classes. You cannot wrap a non-instantiated template .. code-block:: cpp // BROKEN (this will not compile) - py::classh(m, "Cage"); + py::class_(m, "Cage"); .def("get", &Cage::get); You must explicitly specify each template/type combination that you want to @@ -1187,11 +1214,11 @@ wrap separately. .. code-block:: cpp // ok - py::classh>(m, "CatCage") + py::class_>(m, "CatCage") .def("get", &Cage::get); // ok - py::classh>(m, "DogCage") + py::class_>(m, "DogCage") .def("get", &Cage::get); If your class methods have template parameters you can wrap those as well, @@ -1205,7 +1232,7 @@ but once again each instantiation must be explicitly specified: T fn(V v); }; - py::classh>(m, "MyClassT") + py::class>(m, "MyClassT") .def("fn", &MyClass::fn); Custom automatic downcasters @@ -1314,7 +1341,7 @@ Custom type setup For advanced use cases, such as enabling garbage collection support, you may wish to directly manipulate the ``PyHeapTypeObject`` corresponding to a -``py::classh`` definition. +``py::class_`` definition. You can do that using ``py::custom_type_setup``: @@ -1323,7 +1350,7 @@ You can do that using ``py::custom_type_setup``: struct OwnsPythonObjects { py::object value = py::none(); }; - py::classh cls( + py::class_ cls( m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 4313c1b41d..ff00c9c8ac 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -110,7 +110,7 @@ Return value policies can also be applied to properties: .. code-block:: cpp - py::classh(m, "MyClass") + class_(m, "MyClass") .def_property("data", &MyClass::getData, &MyClass::setData, py::return_value_policy::copy); @@ -121,7 +121,7 @@ targeted arguments can be passed through the :class:`cpp_function` constructor: .. code-block:: cpp - py::classh(m, "MyClass") + class_(m, "MyClass") .def_property("data", py::cpp_function(&MyClass::getData, py::return_value_policy::copy), py::cpp_function(&MyClass::setData) @@ -194,7 +194,7 @@ container: .. code-block:: cpp - py::classh(m, "List") + py::class_(m, "List") .def("append", &List::append, py::keep_alive<1, 2>()); For consistency, the argument indexing is identical for constructors. Index @@ -205,7 +205,7 @@ ties the lifetime of the constructor element to the constructed object: .. code-block:: cpp - py::classh(m, "Nurse") + py::class_(m, "Nurse") .def(py::init(), py::keep_alive<1, 2>()); .. note:: @@ -332,11 +332,11 @@ Consider the following example: .. code-block:: cpp - py::classh("MyClass") + py::class_("MyClass") .def("myFunction", py::arg("arg") = SomeType(123)); In this case, pybind11 must already be set up to deal with values of the type -``SomeType`` (via a prior instantiation of ``py::classh``), or an +``SomeType`` (via a prior instantiation of ``py::class_``), or an exception will be thrown. Another aspect worth highlighting is that the "preview" of the default argument @@ -357,7 +357,7 @@ default argument manually using the ``arg_v`` notation: .. code-block:: cpp - py::classh("MyClass") + py::class_("MyClass") .def("myFunction", py::arg_v("arg", SomeType(123), "SomeType(123)")); Sometimes it may be necessary to pass a null pointer value as a default @@ -366,7 +366,7 @@ like so: .. code-block:: cpp - py::classh("MyClass") + py::class_("MyClass") .def("myFunction", py::arg("arg") = static_cast(nullptr)); .. _keyword_only_arguments: @@ -489,7 +489,7 @@ name, i.e. by specifying ``py::arg().noconvert()``. Allow/Prohibiting None arguments ================================ -When a C++ type registered with :class:`py::classh` is passed as an argument to +When a C++ type registered with :class:`py::class_` is passed as an argument to a function taking the instance as pointer or shared holder (e.g. ``shared_ptr`` or a custom, copyable holder as described in :ref:`smart_pointers`), pybind allows ``None`` to be passed from Python which results in calling the C++ @@ -500,8 +500,8 @@ To explicitly enable or disable this behaviour, using the .. code-block:: cpp - py::classh(m, "Dog").def(py::init<>()); - py::classh(m, "Cat").def(py::init<>()); + py::class_(m, "Dog").def(py::init<>()); + py::class_(m, "Cat").def(py::init<>()); m.def("bark", [](Dog *dog) -> std::string { if (dog) return "woof!"; /* Called with a Dog instance */ else return "(no dog)"; /* Called with None, dog == nullptr */ diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 0ba3108690..a256da54a9 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -80,7 +80,7 @@ could be realized as follows (important changes highlighted): .. code-block:: cpp :emphasize-lines: 8,30,31 - class PyAnimal : public Animal, py::trampoline_self_life_support { + class PyAnimal : public Animal { public: /* Inherit the constructors */ using Animal::Animal; @@ -98,12 +98,12 @@ could be realized as follows (important changes highlighted): }; PYBIND11_MODULE(example, m) { - py::classh animal(m, "Animal"); + py::class_ animal(m, "Animal"); animal .def(py::init<>()) .def("go", &Animal::go); - py::classh(m, "Dog", animal) + py::class_(m, "Dog", animal) .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { @@ -177,30 +177,30 @@ from Section :ref:`inheritance`. .. code-block:: cpp - py::classh pet(m, "Pet"); + py::class_ pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name); - py::classh(m, "Dog", pet /* <- specify parent */) + py::class_(m, "Dog", pet /* <- specify parent */) .def(py::init()) .def("bark", &Dog::bark); Suppose now that ``Pet`` bindings are defined in a module named ``basic``, whereas the ``Dog`` bindings are defined somewhere else. The challenge is of course that the variable ``pet`` is not available anymore though it is needed -to indicate the inheritance relationship to the constructor of -``py::classh``. However, it can be acquired as follows: +to indicate the inheritance relationship to the constructor of ``class_``. +However, it can be acquired as follows: .. code-block:: cpp py::object pet = (py::object) py::module_::import("basic").attr("Pet"); - py::classh(m, "Dog", pet) + py::class_(m, "Dog", pet) .def(py::init()) .def("bark", &Dog::bark); Alternatively, you can specify the base class as a template parameter option to -``py::classh``, which performs an automated lookup of the corresponding Python +``class_``, which performs an automated lookup of the corresponding Python type. Like the above code, however, this also requires invoking the ``import`` function once to ensure that the pybind11 binding code of the module ``basic`` has been executed: @@ -209,7 +209,7 @@ has been executed: py::module_::import("basic"); - py::classh(m, "Dog") + py::class_(m, "Dog") .def(py::init()) .def("bark", &Dog::bark); @@ -382,7 +382,7 @@ Avoiding C++ types in docstrings Docstrings are generated at the time of the declaration, e.g. when ``.def(...)`` is called. At this point parameter and return types should be known to pybind11. -If a custom type is not exposed yet through a ``py::classh`` constructor or a custom type caster, +If a custom type is not exposed yet through a ``py::class_`` constructor or a custom type caster, its C++ type name will be used instead to generate the signature in the docstring: .. code-block:: text @@ -399,8 +399,8 @@ before they are used as a parameter or return type of a function: PYBIND11_MODULE(example, m) { - auto pyFoo = py::classh(m, "Foo"); - auto pyBar = py::classh(m, "Bar"); + auto pyFoo = py::class_(m, "Foo"); + auto pyBar = py::class_(m, "Bar"); pyFoo.def(py::init()); pyBar.def(py::init()); diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index f734b0bdee..d09a2cea2c 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -33,7 +33,7 @@ completely avoid copy operations with Python expressions like .. code-block:: cpp - py::classh(m, "Matrix", py::buffer_protocol()) + py::class_(m, "Matrix", py::buffer_protocol()) .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( m.data(), /* Pointer to buffer */ @@ -47,7 +47,7 @@ completely avoid copy operations with Python expressions like }); Supporting the buffer protocol in a new type involves specifying the special -``py::buffer_protocol()`` tag in the ``py::classh`` constructor and calling the +``py::buffer_protocol()`` tag in the ``py::class_`` constructor and calling the ``def_buffer()`` method with a lambda function that creates a ``py::buffer_info`` description record on demand describing a given matrix instance. The contents of ``py::buffer_info`` mirror the Python buffer protocol @@ -80,7 +80,7 @@ buffer objects (e.g. a NumPy matrix). typedef Matrix::Scalar Scalar; constexpr bool rowMajor = Matrix::Flags & Eigen::RowMajorBit; - py::classh(m, "Matrix", py::buffer_protocol()) + py::class_(m, "Matrix", py::buffer_protocol()) .def(py::init([](py::buffer b) { typedef Eigen::Stride Strides; diff --git a/docs/classes.rst b/docs/classes.rst index 6741d73a57..4ae47053c0 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -28,13 +28,13 @@ The binding code for ``Pet`` looks as follows: namespace py = pybind11; PYBIND11_MODULE(example, m) { - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def("setName", &Pet::setName) .def("getName", &Pet::getName); } -``py::classh`` creates bindings for a C++ *class* or *struct*-style data +:class:`class_` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding constructor (see the :ref:`custom_constructors` section for details). @@ -98,7 +98,7 @@ Lambda function instead: .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def("setName", &Pet::setName) .def("getName", &Pet::getName) @@ -129,7 +129,7 @@ method also exists for ``const`` fields. .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def_readwrite("name", &Pet::name) // ... remainder ... @@ -166,7 +166,7 @@ the setter and getter functions: .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def_property("name", &Pet::getName, &Pet::setName) // ... remainder ... @@ -202,7 +202,7 @@ or :func:`class_::def_property`. .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init<>()) .def_readwrite("name", &Pet::name); @@ -220,7 +220,7 @@ must be added to the :class:`py::class_` constructor: .. code-block:: cpp - py::classh(m, "Pet", py::dynamic_attr()) + py::class_(m, "Pet", py::dynamic_attr()) .def(py::init<>()) .def_readwrite("name", &Pet::name); @@ -268,12 +268,12 @@ parameter of the :class:`class_`: .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def_readwrite("name", &Pet::name); // Method 1: template parameter: - py::classh(m, "Dog") + py::class_(m, "Dog") .def(py::init()) .def("bark", &Dog::bark); @@ -282,12 +282,12 @@ Alternatively, we can also assign a name to the previously bound ``Pet`` .. code-block:: cpp - py::classh pet(m, "Pet"); + py::class_ pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name); - // Method 2: pass parent py::classh object: - py::classh(m, "Dog", pet /* <- specify Python parent type */) + // Method 2: pass parent class_ object: + py::class_(m, "Dog", pet /* <- specify Python parent type */) .def(py::init()) .def("bark", &Dog::bark); @@ -334,8 +334,8 @@ will automatically recognize this: }; // Same binding code - py::classh(m, "PolymorphicPet"); - py::classh(m, "PolymorphicDog") + py::class_(m, "PolymorphicPet"); + py::class_(m, "PolymorphicDog") .def(py::init<>()) .def("bark", &PolymorphicDog::bark); @@ -387,7 +387,7 @@ sequence. .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def(py::init()) .def("set", static_cast(&Pet::set), "Set the pet's age") .def("set", static_cast(&Pet::set), "Set the pet's name"); @@ -418,7 +418,7 @@ syntax to cast the overloaded function: .. code-block:: cpp - py::classh(m, "Pet") + py::class_(m, "Pet") .def("set", py::overload_cast(&Pet::set), "Set the pet's age") .def("set", py::overload_cast(&Pet::set), "Set the pet's name"); @@ -434,7 +434,7 @@ on constness, the ``py::const_`` tag should be used: int foo(int x, float y) const; }; - py::classh(m, "Widget") + py::class_(m, "Widget") .def("foo_mutable", py::overload_cast(&Widget::foo)) .def("foo_const", py::overload_cast(&Widget::foo, py::const_)); @@ -446,7 +446,7 @@ you can use ``py::detail::overload_cast_impl`` with an additional set of parenth template using overload_cast_ = pybind11::detail::overload_cast_impl; - py::classh(m, "Pet") + py::class_(m, "Pet") .def("set", overload_cast_()(&Pet::set), "Set the pet's age") .def("set", overload_cast_()(&Pet::set), "Set the pet's name"); @@ -486,7 +486,7 @@ The binding code for this example looks as follows: .. code-block:: cpp - py::classh pet(m, "Pet"); + py::class_ pet(m, "Pet"); pet.def(py::init()) .def_readwrite("name", &Pet::name) @@ -498,7 +498,7 @@ The binding code for this example looks as follows: .value("Cat", Pet::Kind::Cat) .export_values(); - py::classh(pet, "Attributes") + py::class_(pet, "Attributes") .def(py::init<>()) .def_readwrite("age", &Pet::Attributes::age); From 7dc507eeb15f8ea6c7139a7f4c0cd08719525d3b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 28 Feb 2025 17:08:18 -0500 Subject: [PATCH 20/25] docs: focus on py::smart_holder instead of py::classh Signed-off-by: Henry Schreiner --- docs/advanced/cast/stl.rst | 4 ++-- docs/advanced/classes.rst | 13 +++++++------ docs/advanced/misc.rst | 2 +- docs/advanced/smart_ptrs.rst | 17 +++++++++-------- docs/classes.rst | 5 +++-- docs/reference.rst | 2 +- include/pybind11/cast.h | 6 +++--- tests/test_class.py | 2 +- 8 files changed, 27 insertions(+), 24 deletions(-) diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index 42b85532d8..1e17bc389c 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -169,8 +169,8 @@ macro must be specified at the top level (and outside of any namespaces), since it adds a template instantiation of ``type_caster``. If your binding code consists of multiple compilation units, it must be present in every file (typically via a common header) preceding any usage of ``std::vector``. Opaque types must -also have a corresponding ``class_`` declaration to associate them with a name -in Python, and to define a set of available operations, e.g.: +also have a corresponding ``py::class_`` declaration to associate them with a +name in Python, and to define a set of available operations, e.g.: .. code-block:: cpp diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 9fbcc96d5b..f9ea08c02c 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -139,7 +139,7 @@ classes. To enable safely passing a ``std::unique_ptr`` to a trampoline object between Python and C++, -1. the C++ type (``Animal`` above) must be wrapped with ``py::classh`` +1. the C++ type (``Animal`` above) must be wrapped with ``py::class<..., py::smart_holder>`` (see :ref:`smart_holder`), and 2. the trampoline helper class must inherit from @@ -155,7 +155,7 @@ I.e. the example above needs these two changes: .. code-block:: cpp - py::classh(m, "Animal"); + py::class_(m, "Animal"); .. seealso:: @@ -587,9 +587,10 @@ pybind11. The underlying issue is that the ``std::unique_ptr`` holder type that is responsible for managing the lifetime of instances will reference the destructor even if no deallocations ever take place. In order to expose classes with private or protected destructors, it is possible to override the holder -type via a holder type argument to ``class_``. Pybind11 provides a helper class -``py::nodelete`` that disables any destructor invocations. In this case, it is -crucial that instances are deallocated on the C++ side to avoid memory leaks. +type via a holder type argument to ``py::class_``. Pybind11 provides a helper +class ``py::nodelete`` that disables any destructor invocations. In this case, +it is crucial that instances are deallocated on the C++ side to avoid memory +leaks. .. code-block:: cpp @@ -908,7 +909,7 @@ Multiple Inheritance pybind11 can create bindings for types that derive from multiple base types (aka. *multiple inheritance*). To do so, specify all bases in the template -arguments of the ``class_`` declaration: +arguments of the ``py::class_`` declaration: .. code-block:: cpp diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index a256da54a9..510a3a1458 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -200,7 +200,7 @@ However, it can be acquired as follows: .def("bark", &Dog::bark); Alternatively, you can specify the base class as a template parameter option to -``class_``, which performs an automated lookup of the corresponding Python +``py::class_``, which performs an automated lookup of the corresponding Python type. Like the above code, however, this also requires invoking the ``import`` function once to ensure that the pybind11 binding code of the module ``basic`` has been executed: diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 9449d7d626..74441d9016 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -23,19 +23,20 @@ backward compatibility it is **not** the default holder, and there are no plans to make it the default holder in the future. It is extremely easy to use the safer and more versatile ``py::smart_holder``: -simply change +simply add ``py::smart_holder`` to ``py::class_``: * ``py::class_`` to -* ``py::classh``. +* ``py::class_``. .. note:: - ``py::classh`` is a shortcut for ``py::class_``. - — The ``h`` in ``py::classh`` comes from **smart_holder** but is condensed - for brevity. + A shorthand, ``py::classh``, is provided for ``py::class_``. The ``h`` in ``py::classh`` comes from + **smart_holder** but is condensed for brevity; it is the same number of + characters as ``py::class_``. -The ``py::classh`` functionality includes the following: +The ``py::smart_holder`` functionality includes the following: * Support for **two-way** Python/C++ conversions for both ``std::unique_ptr`` and ``std::shared_ptr`` **simultaneously**. @@ -72,7 +73,7 @@ For example, the following code works as expected with ``py::class_``: m.def("create_example", &create_example); However, this will fail with ``py::class_`` (but works with -``py::classh``): +``py::class_``): .. code-block:: cpp @@ -95,7 +96,7 @@ It is possible to use ``std::shared_ptr`` as the holder, for example: py::class_ /* <- holder type */>(m, "Example"); -Compared to using ``py::classh``, there are two noteworthy disadvantages: +Compared to using ``py::class_``, there are two noteworthy disadvantages: * Because a ``py::class_`` for a given C++ type ``T`` can only use a single holder type, ``std::unique_ptr`` cannot even be passed from C++ diff --git a/docs/classes.rst b/docs/classes.rst index 4ae47053c0..22e1fa9887 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -41,8 +41,9 @@ constructor (see the :ref:`custom_constructors` section for details). .. note:: - Starting with pybind11v3, it is recommended to use `py::classh` in most - situations. See :ref:`smart_holder` for more information. + Starting with pybind11v3, it is recommended to include `py::smart_holder` + if you plan to support conversions to C++ smart pointers. See + :ref:`smart_holder` for more information. An interactive Python session demonstrating this example is shown below: diff --git a/docs/reference.rst b/docs/reference.rst index e64a03519d..e351d2b9a1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -68,7 +68,7 @@ Convenience functions converting to Python types .. _extras: -Passing extra arguments to ``def`` or ``class_`` +Passing extra arguments to ``def`` or ``py::class_`` ================================================ .. doxygengroup:: annotations diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 13a589f179..47575084c8 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1062,9 +1062,9 @@ struct move_only_holder_caster< value = sh_load_helper.get_void_ptr_or_nullptr(); return; } - pybind11_fail( - "Passing `std::unique_ptr` from Python to C++ requires `py::classh` (with T = " - + clean_type_id(typeinfo->cpptype->name()) + ")"); + pybind11_fail("Passing `std::unique_ptr` from Python to C++ requires `py::class_` (with T = " + + clean_type_id(typeinfo->cpptype->name()) + ")"); } template diff --git a/tests/test_class.py b/tests/test_class.py index b8102ec63b..2e11feb7b5 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -54,7 +54,7 @@ def test_pass_unique_ptr(): with pytest.raises(RuntimeError) as execinfo: m.pass_unique_ptr(obj) assert str(execinfo.value).startswith( - "Passing `std::unique_ptr` from Python to C++ requires `py::classh` (with T = " + "Passing `std::unique_ptr` from Python to C++ requires `py::class_` (with T = " ) assert "ToBeHeldByUniquePtr" in str(execinfo.value) From 8409b19131e995930ae1c969d20e88b21a0363d8 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 28 Feb 2025 20:16:53 -0800 Subject: [PATCH 21/25] Restore minor general fixes that got lost when ac9d31e13fd15f2a8e3deeccd840e5038c028da8 was reverted. --- docs/advanced/classes.rst | 20 ++++++++++---------- docs/advanced/misc.rst | 2 +- docs/classes.rst | 8 ++++---- docs/reference.rst | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index f9ea08c02c..1a1648d338 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -117,7 +117,7 @@ The binding code also needs a few minor adaptations (highlighted): } Importantly, pybind11 is made aware of the trampoline helper class by -specifying it as an extra template argument to :class:`class_`. (This can also +specifying it as an extra template argument to ``py::class_``. (This can also be combined with other template arguments such as a custom holder type; the order of template types does not matter). Following this, we are able to define a constructor as usual. @@ -139,8 +139,8 @@ classes. To enable safely passing a ``std::unique_ptr`` to a trampoline object between Python and C++, -1. the C++ type (``Animal`` above) must be wrapped with ``py::class<..., py::smart_holder>`` - (see :ref:`smart_holder`), and +1. the C++ type (``Animal`` above) must be wrapped with + ``py::class_<..., py::smart_holder>`` (see :ref:`smart_holder`), and 2. the trampoline helper class must inherit from ``py::trampoline_self_life_support``. @@ -984,11 +984,11 @@ because of conflicting definitions on the external type: // dogs.cpp // Binding for external library class: - py::class(m, "Pet") + py::class_(m, "Pet") .def("name", &pets::Pet::name); // Binding for local extension class: - py::class(m, "Dog") + py::class_(m, "Dog") .def(py::init()); .. code-block:: cpp @@ -996,11 +996,11 @@ because of conflicting definitions on the external type: // cats.cpp, in a completely separate project from the above dogs.cpp. // Binding for external library class: - py::class(m, "Pet") + py::class_(m, "Pet") .def("get_name", &pets::Pet::name); // Binding for local extending class: - py::class(m, "Cat") + py::class_(m, "Cat") .def(py::init()); .. code-block:: pycon @@ -1018,13 +1018,13 @@ the ``py::class_`` constructor: .. code-block:: cpp // Pet binding in dogs.cpp: - py::class(m, "Pet", py::module_local()) + py::class_(m, "Pet", py::module_local()) .def("name", &pets::Pet::name); .. code-block:: cpp // Pet binding in cats.cpp: - py::class(m, "Pet", py::module_local()) + py::class_(m, "Pet", py::module_local()) .def("get_name", &pets::Pet::name); This makes the Python-side ``dogs.Pet`` and ``cats.Pet`` into distinct classes, @@ -1233,7 +1233,7 @@ but once again each instantiation must be explicitly specified: T fn(V v); }; - py::class>(m, "MyClassT") + py::class_>(m, "MyClassT") .def("fn", &MyClass::fn); Custom automatic downcasters diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 510a3a1458..a0438f0330 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -188,7 +188,7 @@ from Section :ref:`inheritance`. Suppose now that ``Pet`` bindings are defined in a module named ``basic``, whereas the ``Dog`` bindings are defined somewhere else. The challenge is of course that the variable ``pet`` is not available anymore though it is needed -to indicate the inheritance relationship to the constructor of ``class_``. +to indicate the inheritance relationship to the constructor of ``py::class_``. However, it can be acquired as follows: .. code-block:: cpp diff --git a/docs/classes.rst b/docs/classes.rst index 22e1fa9887..07aded277e 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -34,7 +34,7 @@ The binding code for ``Pet`` looks as follows: .def("getName", &Pet::getName); } -:class:`class_` creates bindings for a C++ *class* or *struct*-style data +``py::class_`` creates bindings for a C++ *class* or *struct*-style data structure. :func:`init` is a convenience function that takes the types of a constructor's parameters as template arguments and wraps the corresponding constructor (see the :ref:`custom_constructors` section for details). @@ -265,7 +265,7 @@ inheritance relationship: There are two different ways of indicating a hierarchical relationship to pybind11: the first specifies the C++ base class as an extra template -parameter of the :class:`class_`: +parameter of the ``py::class_``: .. code-block:: cpp @@ -279,7 +279,7 @@ parameter of the :class:`class_`: .def("bark", &Dog::bark); Alternatively, we can also assign a name to the previously bound ``Pet`` -:class:`class_` object and reference it when binding the ``Dog`` class: +``py::class_`` object and reference it when binding the ``Dog`` class: .. code-block:: cpp @@ -505,7 +505,7 @@ The binding code for this example looks as follows: To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the -``pet`` :class:`class_` instance must be supplied to the :class:`enum_` and :class:`class_` +``pet`` ``py::class_`` instance must be supplied to the :class:`enum_` and ``py::class_`` constructor. The :func:`enum_::export_values` function exports the enum entries into the parent scope, which should be skipped for newer C++11-style strongly typed enums. diff --git a/docs/reference.rst b/docs/reference.rst index e351d2b9a1..c2757988d2 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -69,7 +69,7 @@ Convenience functions converting to Python types .. _extras: Passing extra arguments to ``def`` or ``py::class_`` -================================================ +==================================================== .. doxygengroup:: annotations :members: From 85cc92d5d1a26d0f99f239b0afacba10fc313945 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 28 Feb 2025 20:23:39 -0800 Subject: [PATCH 22/25] Remove `- smart_holder` from list of branches in all .github/workflows --- .github/workflows/ci.yml | 1 - .github/workflows/configure.yml | 1 - .github/workflows/emscripten.yaml | 1 - .github/workflows/format.yml | 1 - .github/workflows/pip.yml | 1 - 5 files changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5c75146f0..80f1f0d74b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,6 @@ on: branches: - master - stable - - smart_holder - v* permissions: read-all diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 8e89e6f394..2031ec8236 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -7,7 +7,6 @@ on: branches: - master - stable - - smart_holder - v* permissions: diff --git a/.github/workflows/emscripten.yaml b/.github/workflows/emscripten.yaml index c7fd73cdf3..5eac089e5c 100644 --- a/.github/workflows/emscripten.yaml +++ b/.github/workflows/emscripten.yaml @@ -6,7 +6,6 @@ on: branches: - master - stable - - smart_holder - v* concurrency: diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 6bb22171b7..e50dc0bb72 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -10,7 +10,6 @@ on: branches: - master - stable - - smart_holder - "v*" permissions: diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index a32e7fe8f1..c10087eff4 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -7,7 +7,6 @@ on: branches: - master - stable - - smart_holder - v* release: types: From b73f430d07cae01ddc231654d07cc86a700e5160 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 28 Feb 2025 20:29:49 -0800 Subject: [PATCH 23/25] Extend classh note to explain whitespace noise motivation. --- docs/advanced/smart_ptrs.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index 74441d9016..599f384db0 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -31,10 +31,12 @@ simply add ``py::smart_holder`` to ``py::class_``: .. note:: - A shorthand, ``py::classh``, is provided for ``py::class_``. The ``h`` in ``py::classh`` comes from - **smart_holder** but is condensed for brevity; it is the same number of - characters as ``py::class_``. + A shorthand, ``py::classh``, is provided for + ``py::class_``. The ``h`` in ``py::classh`` stands + for **smart_holder** but is shortened for brevity, ensuring it has the + same number of characters as ``py::class_``. This design choice facilitates + easy experimentation with ``py::smart_holder`` without introducing + distracting whitespace noise in diffs. The ``py::smart_holder`` functionality includes the following: From d3f9f93ab7eba799fcd9e08b68f3363d54042074 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 28 Feb 2025 20:30:23 -0800 Subject: [PATCH 24/25] Suggest `py::smart_holder` for "most situations for safety" --- docs/classes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/classes.rst b/docs/classes.rst index 07aded277e..5406668f0b 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -42,8 +42,8 @@ constructor (see the :ref:`custom_constructors` section for details). .. note:: Starting with pybind11v3, it is recommended to include `py::smart_holder` - if you plan to support conversions to C++ smart pointers. See - :ref:`smart_holder` for more information. + in most situations for safety, especially if you plan to support conversions + to C++ smart pointers. See :ref:`smart_holder` for more information. An interactive Python session demonstrating this example is shown below: From 4b0564337653e084bee38512be8fbe570500a475 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 1 Mar 2025 09:30:46 -0800 Subject: [PATCH 25/25] Add back PYBIND11_HAS_INTERNALS_WITH_SMART_HOLDER_SUPPORT This define was * introduced with https://github.com/pybind/pybind11/pull/5286 * removed with https://github.com/pybind/pybind11/pull/5531 It is has been in use here: * https://github.com/pybind/pybind11_protobuf/blob/f02a2b7653bc50eb5119d125842a3870db95d251/pybind11_protobuf/native_proto_caster.h#L89-L101 Currently pybind11 unit tests for the two holder caster backwards compatibility traits * `copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled` * `move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled` are missing. --- include/pybind11/detail/internals.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 234524f910..841c8fe155 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -211,6 +211,9 @@ struct internals { } }; +// For backwards compatibility (i.e. #ifdef guards): +#define PYBIND11_HAS_INTERNALS_WITH_SMART_HOLDER_SUPPORT + enum class holder_enum_t : uint8_t { undefined, std_unique_ptr, // Default, lacking interop with std::shared_ptr.