pythongh-59956: Clarify Runtime State Status Expectations (pythongh-1…

…01308)

A PyThreadState can be in one of many states in its lifecycle, represented by some status value.  Those statuses haven't been particularly clear, so we're addressing that here.  Specifically:

* made the distinct lifecycle statuses clear on PyThreadState
* identified expectations of how various lifecycle-related functions relate to status
* noted the various places where those expectations don't match the actual behavior

At some point we'll need to address the mismatches.

(This change also includes some cleanup.)

python#59956

Include/cpython/pystate.h

@@ -119,7 +119,30 @@ struct _ts {
     PyThreadState *next;
     PyInterpreterState *interp;
 
-    int _status;
+    struct {
+        /* Has been initialized to a safe state.
+
+           In order to be effective, this must be set to 0 during or right
+           after allocation. */
+        unsigned int initialized:1;
+
+        /* Has been bound to an OS thread. */
+        unsigned int bound:1;
+        /* Has been unbound from its OS thread. */
+        unsigned int unbound:1;
+        /* Has been bound aa current for the GILState API. */
+        unsigned int bound_gilstate:1;
+        /* Currently in use (maybe holds the GIL). */
+        unsigned int active:1;
+
+        /* various stages of finalization */
+        unsigned int finalizing:1;
+        unsigned int cleared:1;
+        unsigned int finalized:1;
+
+        /* padding to align to 4 bytes */
+        unsigned int :24;
+    } _status;
 
     int py_recursion_remaining;
     int py_recursion_limit;
@@ -245,6 +268,8 @@ struct _ts {
 // Alias for backward compatibility with Python 3.8
 #define _PyInterpreterState_Get PyInterpreterState_Get
 
+/* An alias for the internal _PyThreadState_New(),
+   kept for stable ABI compatibility. */
 PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *);
 
 /* Similar to PyThreadState_Get(), but don't issue a fatal error

Include/internal/pycore_pystate.h

@@ -120,13 +120,12 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) {
 
 // PyThreadState functions
 
+PyAPI_FUNC(PyThreadState *) _PyThreadState_New(PyInterpreterState *interp);
 PyAPI_FUNC(void) _PyThreadState_Bind(PyThreadState *tstate);
 // We keep this around exclusively for stable ABI compatibility.
 PyAPI_FUNC(void) _PyThreadState_Init(
     PyThreadState *tstate);
-PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
-    _PyRuntimeState *runtime,
-    PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
 
 
 static inline void
@@ -139,18 +138,6 @@ _PyThreadState_UpdateTracingState(PyThreadState *tstate)
 }
 
 
-/* PyThreadState status */
-
-#define PyThreadState_UNINITIALIZED 0
-/* Has been initialized to a safe state.
-
-   In order to be effective, this must be set to 0 during or right
-   after allocation. */
-#define PyThreadState_INITIALIZED 1
-#define PyThreadState_BOUND 2
-#define PyThreadState_UNBOUND 3
-
-
 /* Other */
 
 PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(

Modules/_threadmodule.c

@@ -1161,7 +1161,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
         return PyErr_NoMemory();
     }
     boot->interp = _PyInterpreterState_GET();
-    boot->tstate = _PyThreadState_Prealloc(boot->interp);
+    boot->tstate = _PyThreadState_New(boot->interp);
     if (boot->tstate == NULL) {
         PyMem_Free(boot);
         if (!PyErr_Occurred()) {

Python/ceval_gil.c

@@ -624,7 +624,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate)
     }
 
     /* Destroy all threads except the current one */
-    _PyThreadState_DeleteExcept(runtime, tstate);
+    _PyThreadState_DeleteExcept(tstate);
     return _PyStatus_OK();
 }
 #endif

Python/pylifecycle.c

@@ -696,10 +696,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
     const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
     init_interp_settings(interp, &config);
 
-    PyThreadState *tstate = PyThreadState_New(interp);
+    PyThreadState *tstate = _PyThreadState_New(interp);
     if (tstate == NULL) {
         return _PyStatus_ERR("can't make first thread");
     }
+    _PyThreadState_Bind(tstate);
     (void) PyThreadState_Swap(tstate);
 
     status = init_interp_create_gil(tstate);
@@ -1821,6 +1822,11 @@ Py_FinalizeEx(void)
 
     /* Get current thread state and interpreter pointer */
     PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    // XXX assert(_Py_IsMainInterpreter(tstate->interp));
+    // XXX assert(_Py_IsMainThread());
+
+    // Block some operations.
+    tstate->interp->finalizing = 1;
 
     // Wrap up existing "threading"-module-created, non-daemon threads.
     wait_for_thread_shutdown(tstate);
@@ -1867,7 +1873,23 @@ Py_FinalizeEx(void)
        _PyRuntimeState_SetFinalizing() has been called, no other Python thread
        can take the GIL at this point: if they try, they will exit
        immediately. */
-    _PyThreadState_DeleteExcept(runtime, tstate);
+    _PyThreadState_DeleteExcept(tstate);
+
+    /* At this point no Python code should be running at all.
+       The only thread state left should be the main thread of the main
+       interpreter (AKA tstate), in which this code is running right now.
+       There may be other OS threads running but none of them will have
+       thread states associated with them, nor will be able to create
+       new thread states.
+
+       Thus tstate is the only possible thread state from here on out.
+       It may still be used during finalization to run Python code as
+       needed or provide runtime state (e.g. sys.modules) but that will
+       happen sparingly.  Furthermore, the order of finalization aims
+       to not need a thread (or interpreter) state as soon as possible.
+     */
+    // XXX Make sure we are preventing the creating of any new thread states
+    // (or interpreters).
 
     /* Flush sys.stdout and sys.stderr */
     if (flush_std_files() < 0) {
@@ -1958,6 +1980,20 @@ Py_FinalizeEx(void)
     }
 #endif /* Py_TRACE_REFS */
 
+    /* At this point there's almost no other Python code that will run,
+       nor interpreter state needed.  The only possibility is the
+       finalizers of the objects stored on tstate (and tstate->interp),
+       which are triggered via finalize_interp_clear().
+
+       For now we operate as though none of those finalizers actually
+       need an operational thread state or interpreter.  In reality,
+       those finalizers may rely on some part of tstate or
+       tstate->interp, and/or may raise exceptions
+       or otherwise fail.
+     */
+    // XXX Do this sooner during finalization.
+    // XXX Ensure finalizer errors are handled properly.
+
     finalize_interp_clear(tstate);
     finalize_interp_delete(tstate->interp);
 
@@ -2039,12 +2075,13 @@ new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config)
         return _PyStatus_OK();
     }
 
-    PyThreadState *tstate = PyThreadState_New(interp);
+    PyThreadState *tstate = _PyThreadState_New(interp);
     if (tstate == NULL) {
         PyInterpreterState_Delete(interp);
         *tstate_p = NULL;
         return _PyStatus_OK();
     }
+    _PyThreadState_Bind(tstate);
 
     PyThreadState *save_tstate = PyThreadState_Swap(tstate);