pythongh-128844: Make _Py_TryIncref public as an unstable API. (pyt…

…hon#128926)

This exposes `_Py_TryIncref` as `PyUnstable_TryIncref()` and the helper
function `_PyObject_SetMaybeWeakref` as `PyUnstable_EnableTryIncRef`.

These are helpers for dealing with unowned references in a safe way,
particularly in the free threading build.

Co-authored-by: Petr Viktorin <[email protected]>

Doc/c-api/object.rst

@@ -624,3 +624,84 @@ Object Protocol
       be immortal in another.
 
    .. versionadded:: next
+
+.. c:function:: int PyUnstable_TryIncRef(PyObject *obj)
+
+   Increments the reference count of *obj* if it is not zero.  Returns ``1``
+   if the object's reference count was successfully incremented. Otherwise,
+   this function returns ``0``.
+
+   :c:func:`PyUnstable_EnableTryIncRef` must have been called
+   earlier on *obj* or this function may spuriously return ``0`` in the
+   :term:`free threading` build.
+
+   This function is logically equivalent to the following C code, except that
+   it behaves atomically in the :term:`free threading` build::
+
+      if (Py_REFCNT(op) > 0) {
+         Py_INCREF(op);
+         return 1;
+      }
+      return 0;
+
+   This is intended as a building block for managing weak references
+   without the overhead of a Python :ref:`weak reference object <weakrefobjects>`.
+
+   Typically, correct use of this function requires support from *obj*'s
+   deallocator (:c:member:`~PyTypeObject.tp_dealloc`).
+   For example, the following sketch could be adapted to implement a
+   "weakmap" that works like a :py:class:`~weakref.WeakValueDictionary`
+   for a specific type:
+
+   .. code-block:: c
+
+      PyMutex mutex;
+
+      PyObject *
+      add_entry(weakmap_key_type *key, PyObject *value)
+      {
+          PyUnstable_EnableTryIncRef(value);
+          weakmap_type weakmap = ...;
+          PyMutex_Lock(&mutex);
+          weakmap_add_entry(weakmap, key, value);
+          PyMutex_Unlock(&mutex);
+          Py_RETURN_NONE;
+      }
+
+      PyObject *
+      get_value(weakmap_key_type *key)
+      {
+          weakmap_type weakmap = ...;
+          PyMutex_Lock(&mutex);
+          PyObject *result = weakmap_find(weakmap, key);
+          if (PyUnstable_TryIncRef(result)) {
+              // `result` is safe to use
+              PyMutex_Unlock(&mutex);
+              return result;
+          }
+          // if we get here, `result` is starting to be garbage-collected,
+          // but has not been removed from the weakmap yet
+          PyMutex_Unlock(&mutex);
+          return NULL;
+      }
+
+      // tp_dealloc function for weakmap values
+      void
+      value_dealloc(PyObject *value)
+      {
+          weakmap_type weakmap = ...;
+          PyMutex_Lock(&mutex);
+          weakmap_remove_value(weakmap, value);
+
+          ...
+          PyMutex_Unlock(&mutex);
+      }
+
+   .. versionadded:: 3.14
+
+.. c:function:: void PyUnstable_EnableTryIncRef(PyObject *obj)
+
+   Enables subsequent uses of :c:func:`PyUnstable_TryIncRef` on *obj*.  The
+   caller must hold a :term:`strong reference` to *obj* when calling this.
+
+   .. versionadded:: 3.14

Include/cpython/object.h

@@ -544,3 +544,9 @@ PyAPI_FUNC(int) PyUnstable_Object_EnableDeferredRefcount(PyObject *);
 
 /* Check whether the object is immortal. This cannot fail. */
 PyAPI_FUNC(int) PyUnstable_IsImmortal(PyObject *);
+
+// Increments the reference count of the object, if it's not zero.
+// PyUnstable_EnableTryIncRef() should be called on the object
+// before calling this function in order to avoid spurious failures.
+PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *);
+PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *);

Misc/NEWS.d/next/C_API/2025-01-16-21-56-49.gh-issue-128844.ZPiJuo.rst

@@ -0,0 +1,3 @@
+Add :c:func:`PyUnstable_TryIncRef` and :c:func:`PyUnstable_EnableTryIncRef`
+unstable APIs.  These are helpers for dealing with unowned references in
+a thread-safe way, particularly in the free threading build.

Modules/_testcapi/object.c

@@ -131,6 +131,59 @@ pyobject_enable_deferred_refcount(PyObject *self, PyObject *obj)
     return PyLong_FromLong(result);
 }
 
+static int MyObject_dealloc_called = 0;
+
+static void
+MyObject_dealloc(PyObject *op)
+{
+    // PyUnstable_TryIncRef should return 0 if object is being deallocated
+    assert(Py_REFCNT(op) == 0);
+    assert(!PyUnstable_TryIncRef(op));
+    assert(Py_REFCNT(op) == 0);
+
+    MyObject_dealloc_called++;
+    Py_TYPE(op)->tp_free(op);
+}
+
+static PyTypeObject MyType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "MyType",
+    .tp_basicsize = sizeof(PyObject),
+    .tp_dealloc = MyObject_dealloc,
+};
+
+static PyObject *
+test_py_try_inc_ref(PyObject *self, PyObject *unused)
+{
+    if (PyType_Ready(&MyType) < 0) {
+        return NULL;
+    }
+
+    MyObject_dealloc_called = 0;
+
+    PyObject *op = PyObject_New(PyObject, &MyType);
+    if (op == NULL) {
+        return NULL;
+    }
+
+    PyUnstable_EnableTryIncRef(op);
+#ifdef Py_GIL_DISABLED
+    // PyUnstable_EnableTryIncRef sets the shared flags to
+    // `_Py_REF_MAYBE_WEAKREF` if the flags are currently zero to ensure that
+    // the shared reference count is merged on deallocation.
+    assert((op->ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) >= _Py_REF_MAYBE_WEAKREF);
+#endif
+
+    if (!PyUnstable_TryIncRef(op)) {
+        PyErr_SetString(PyExc_AssertionError, "PyUnstable_TryIncRef failed");
+        Py_DECREF(op);
+        return NULL;
+    }
+    Py_DECREF(op);  // undo try-incref
+    Py_DECREF(op);  // dealloc
+    assert(MyObject_dealloc_called == 1);
+    Py_RETURN_NONE;
+}
 
 static PyMethodDef test_methods[] = {
     {"call_pyobject_print", call_pyobject_print, METH_VARARGS},
@@ -139,6 +192,7 @@ static PyMethodDef test_methods[] = {
     {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS},
     {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O},
     {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O},
+    {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS},
     {NULL},
 };
 

Objects/object.c

@@ -2588,6 +2588,20 @@ PyUnstable_Object_EnableDeferredRefcount(PyObject *op)
 #endif
 }
 
+int
+PyUnstable_TryIncRef(PyObject *op)
+{
+    return _Py_TryIncref(op);
+}
+
+void
+PyUnstable_EnableTryIncRef(PyObject *op)
+{
+#ifdef Py_GIL_DISABLED
+    _PyObject_SetMaybeWeakref(op);
+#endif
+}
+
 void
 _Py_ResurrectReference(PyObject *op)
 {

Tools/c-analyzer/cpython/ignored.tsv

@@ -447,6 +447,8 @@ Modules/_testcapi/exceptions.c	-	PyRecursingInfinitelyError_Type	-
 Modules/_testcapi/heaptype.c	-	_testcapimodule	-
 Modules/_testcapi/mem.c	-	FmData	-
 Modules/_testcapi/mem.c	-	FmHook	-
+Modules/_testcapi/object.c	-	MyObject_dealloc_called	-
+Modules/_testcapi/object.c	-	MyType	-
 Modules/_testcapi/structmember.c	-	test_structmembersType_OldAPI	-
 Modules/_testcapi/watchers.c	-	g_dict_watch_events	-
 Modules/_testcapi/watchers.c	-	g_dict_watchers_installed	-