From fe6e8d26c1cbba9a0f44a204367415101da8b166 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 6 Dec 2022 11:08:06 -0700 Subject: [PATCH 1/6] Add _PyRuntimeState.os. --- Include/internal/pycore_os.h | 22 ++++++++++++++++++++++ Include/internal/pycore_runtime.h | 2 ++ Include/internal/pycore_runtime_init.h | 1 + Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ 6 files changed, 30 insertions(+) create mode 100644 Include/internal/pycore_os.h diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h new file mode 100644 index 00000000000000..475ca4beb42f76 --- /dev/null +++ b/Include/internal/pycore_os.h @@ -0,0 +1,22 @@ +#ifndef Py_INTERNAL_OS_H +#define Py_INTERNAL_OS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +struct _os_runtime_state { + int _not_used; +}; + +#define _OS_RUNTIME_INIT {0} + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OS_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0720e2ed4422df..c6b48cae7760fc 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_os.h" // struct _os_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -103,6 +104,7 @@ typedef struct pyruntimestate { * KeyboardInterrupt exception, suggesting the user pressed ^C. */ int unhandled_keyboard_interrupt; } signals; + struct _os_runtime_state os; struct pyinterpreters { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ab53876e355fd8..70263892e570b2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -26,6 +26,7 @@ extern "C" { }, \ .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ .pyhash_state = pyhash_state_INIT, \ + .os = _OS_RUNTIME_INIT, \ .interpreters = { \ /* This prevents interpreters from getting created \ until _PyInterpreterState_Enable() is called. */ \ diff --git a/Makefile.pre.in b/Makefile.pre.in index f6df7a620deaed..80144e5288755c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1654,6 +1654,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ + $(srcdir)/Include/internal/pycore_os.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f62434370cfdf7..fa3924968a6b26 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -236,6 +236,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f44a1ad8550a38..e29c6b2330d176 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -612,6 +612,9 @@ Include\internal + + Include\internal + Include\internal From 9a86dd3d2c160ae05f321c6d58a26196d57cc2b3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 Dec 2022 15:48:16 -0700 Subject: [PATCH 2/6] Move dup3_works to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - Tools/c-analyzer/cpython/ignored.tsv | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5c8164517f138a..ff6e87399403b2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,7 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/posixmodule.c os_dup2_impl dup3_works - Modules/posixmodule.c - structseq_new - Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 257823574fa8b3..814c55b75c090d 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -11,14 +11,18 @@ filename funcname name reason # These are effectively const. ##----------------------- -## process-global resources - set during first init +## process-global resources ## indicators for resource availability/capability +# (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - +# (set lazily, *after* first init) +# XXX Is this thread-safe? +Modules/posixmodule.c os_dup2_impl dup3_works - -## resource init +## resource init - set during first init Python/thread.c - initialized - Python/thread_pthread.h - condattr_monotonic - # safe static buffer used during one-time initialization From 7f6d5b8f6374ece7803cee0c2b88aa41869ee143 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 6 Dec 2022 12:14:32 -0700 Subject: [PATCH 3/6] Move ticks_per_second to _PyRuntimeState. --- Include/internal/pycore_os.h | 20 ++++++++++-- Modules/posixmodule.c | 36 +++++++++++++-------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_os.h b/Include/internal/pycore_os.h index 475ca4beb42f76..e4899bd5032df9 100644 --- a/Include/internal/pycore_os.h +++ b/Include/internal/pycore_os.h @@ -9,11 +9,27 @@ extern "C" { #endif +#ifndef MS_WINDOWS +#define _OS_NEED_TICKS_PER_SECOND +# define need_ticks_per_second_STATE \ + long ticks_per_second; +# define need_ticks_per_second_INIT \ + .ticks_per_second = -1, +#else +# define need_ticks_per_second_STATE +# define need_ticks_per_second_INIT +#endif /* MS_WINDOWS */ + + struct _os_runtime_state { int _not_used; + need_ticks_per_second_STATE }; - -#define _OS_RUNTIME_INIT {0} +# define _OS_RUNTIME_INIT \ + { \ + ._not_used = 0, \ + need_ticks_per_second_INIT \ + } #ifdef __cplusplus diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7fc8aef9b303fc..f01f70965f11a9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9051,10 +9051,23 @@ build_times_result(PyObject *module, double user, double system, } -#ifndef MS_WINDOWS -#define NEED_TICKS_PER_SECOND -static long ticks_per_second = -1; -#endif /* MS_WINDOWS */ +#ifdef _OS_NEED_TICKS_PER_SECOND +#define ticks_per_second _PyRuntime.os.ticks_per_second +static void +ticks_per_second_init(void) +{ + if (ticks_per_second != -1) { + return; + } +# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) + ticks_per_second = sysconf(_SC_CLK_TCK); +# elif defined(HZ) + ticks_per_second = HZ; +# else + ticks_per_second = 60; /* magic fallback value; may be bogus */ +# endif +} +#endif /*[clinic input] os.times @@ -9089,10 +9102,10 @@ os_times_impl(PyObject *module) (double)0, (double)0); } +#elif !defined(_OS_NEED_TICKS_PER_SECOND) +# error "missing ticks_per_second" #else /* MS_WINDOWS */ { - - struct tms t; clock_t c; errno = 0; @@ -15922,14 +15935,9 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "statvfs_result", Py_NewRef(StatVFSResultType)); state->StatVFSResultType = StatVFSResultType; -#ifdef NEED_TICKS_PER_SECOND -# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); -# elif defined(HZ) - ticks_per_second = HZ; -# else - ticks_per_second = 60; /* magic fallback value; may be bogus */ -# endif + +#ifdef _OS_NEED_TICKS_PER_SECOND + ticks_per_second_init(); #endif #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index ff6e87399403b2..afbcc310111656 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -391,7 +391,6 @@ Modules/faulthandler.c - old_stack - ## initialized once Modules/posixmodule.c - structseq_new - -Modules/posixmodule.c - ticks_per_second - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - From 62179ba16f4c35f456c5e39cf1f74ea17ed15d8a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 6 Dec 2022 11:35:06 -0700 Subject: [PATCH 4/6] Eliminate structseq_new. --- .../pycore_global_objects_fini_generated.h | 2 ++ Include/internal/pycore_global_strings.h | 2 ++ Include/internal/pycore_runtime_init_generated.h | 2 ++ Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Modules/posixmodule.c | 14 ++++++++++++-- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 6 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9951fa9951e67a..6aba2f19ebde4a 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1051,6 +1051,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(node_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ns)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nstype)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(number)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(obj)); @@ -1089,6 +1090,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 12144b02f45574..acb9a4fbb92dce 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -537,6 +537,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(node_offset) STRUCT_FOR_ID(ns) STRUCT_FOR_ID(nstype) + STRUCT_FOR_ID(nt) STRUCT_FOR_ID(null) STRUCT_FOR_ID(number) STRUCT_FOR_ID(obj) @@ -575,6 +576,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) + STRUCT_FOR_ID(posix) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 87b0f2ed8dfa8c..6d1b8702c77698 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1043,6 +1043,7 @@ extern "C" { INIT_ID(node_offset), \ INIT_ID(ns), \ INIT_ID(nstype), \ + INIT_ID(nt), \ INIT_ID(null), \ INIT_ID(number), \ INIT_ID(obj), \ @@ -1081,6 +1082,7 @@ extern "C" { INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ + INIT_ID(posix), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 80be342b5b3b44..7f407c0141b8a5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -980,6 +980,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(nstype); PyUnicode_InternInPlace(&string); + string = &_Py_ID(nt); + PyUnicode_InternInPlace(&string); string = &_Py_ID(null); PyUnicode_InternInPlace(&string); string = &_Py_ID(number); @@ -1056,6 +1058,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(pos2); PyUnicode_InternInPlace(&string); + string = &_Py_ID(posix); + PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f01f70965f11a9..71c3ab74a37a93 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -495,9 +495,11 @@ extern char *ctermid_r(char *); #ifdef MS_WINDOWS # define INITFUNC PyInit_nt # define MODNAME "nt" +# define MODNAME_OBJ &_Py_ID(nt) #else # define INITFUNC PyInit_posix # define MODNAME "posix" +# define MODNAME_OBJ &_Py_ID(posix) #endif #if defined(__sun) @@ -2225,7 +2227,6 @@ static PyStructSequence_Desc waitid_result_desc = { 5 }; #endif -static newfunc structseq_new; static PyObject * statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -2233,6 +2234,16 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyStructSequence *result; int i; + PyObject *mod = PyType_GetModule(type); + if (mod == NULL) { + return NULL; + } + _posixstate *state = get_posix_state(mod); + if (state == NULL) { + return NULL; + } +#define structseq_new ((PyTypeObject *)state->StatResultType)->tp_new + result = (PyStructSequence*)structseq_new(type, args, kwds); if (!result) return NULL; @@ -15925,7 +15936,6 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "stat_result", Py_NewRef(StatResultType)); state->StatResultType = StatResultType; - structseq_new = ((PyTypeObject *)StatResultType)->tp_new; ((PyTypeObject *)StatResultType)->tp_new = statresult_new; statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index afbcc310111656..8e05bc3e4f23e5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,7 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/posixmodule.c - structseq_new - Modules/timemodule.c _PyTime_GetClockWithInfo initialized - Modules/timemodule.c _PyTime_GetProcessTimeWithInfo ticks_per_second - From d52f5dd9bf0e00410b15b2247a846a944d29914b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 8 Dec 2022 10:38:40 -0700 Subject: [PATCH 5/6] Use subtests for test_stat_result_pickle. --- Lib/test/test_os.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 94db8bb7737acd..e0577916428a08 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -606,12 +606,13 @@ def test_stat_attributes_bytes(self): def test_stat_result_pickle(self): result = os.stat(self.fname) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - p = pickle.dumps(result, proto) - self.assertIn(b'stat_result', p) - if proto < 4: - self.assertIn(b'cos\nstat_result\n', p) - unpickled = pickle.loads(p) - self.assertEqual(result, unpickled) + with self.subTest(f'protocol {proto}'): + p = pickle.dumps(result, proto) + self.assertIn(b'stat_result', p) + if proto < 4: + self.assertIn(b'cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): From b8befdc09473a49bc8048b72e28e6fa778b7a0ad Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 8 Dec 2022 14:00:21 -0700 Subject: [PATCH 6/6] Store the original tp_new in module state. --- Modules/posixmodule.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 71c3ab74a37a93..cbf4d5b3fcec5e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -976,6 +976,7 @@ typedef struct { #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) PyObject *SchedParamType; #endif + newfunc statresult_new_orig; PyObject *StatResultType; PyObject *StatVFSResultType; PyObject *TerminalSizeType; @@ -2234,7 +2235,9 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyStructSequence *result; int i; - PyObject *mod = PyType_GetModule(type); + // ht_module doesn't get set in PyStructSequence_NewType(), + // so we can't use PyType_GetModule(). + PyObject *mod = PyImport_GetModule(MODNAME_OBJ); if (mod == NULL) { return NULL; } @@ -2242,7 +2245,7 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (state == NULL) { return NULL; } -#define structseq_new ((PyTypeObject *)state->StatResultType)->tp_new +#define structseq_new state->statresult_new_orig result = (PyStructSequence*)structseq_new(type, args, kwds); if (!result) @@ -15936,6 +15939,7 @@ posixmodule_exec(PyObject *m) } PyModule_AddObject(m, "stat_result", Py_NewRef(StatResultType)); state->StatResultType = StatResultType; + state->statresult_new_orig = ((PyTypeObject *)StatResultType)->tp_new; ((PyTypeObject *)StatResultType)->tp_new = statresult_new; statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */