From 2e438d91d6bdd30407db02d0c7fa8aae21a22779 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 3 Jul 2022 01:55:00 +0300 Subject: [PATCH 01/20] Convert fork_exec to pre-inlined-argparser Argument Clinic --- Lib/multiprocessing/util.py | 11 ++- Modules/_posixsubprocess.c | 135 ++++++++++++++++------------ Modules/clinic/_posixsubprocess.c.h | 88 ++++++++++++++++++ 3 files changed, 173 insertions(+), 61 deletions(-) create mode 100644 Modules/clinic/_posixsubprocess.c.h diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 6ee0d33e88a060..cb7b3cd81a079e 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -451,10 +451,13 @@ def spawnv_passfds(path, args, passfds): errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( - args, [path], True, passfds, None, None, - -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, -1, None, None, None, -1, None, - subprocess._USE_VFORK) + args, executable_list=[path], close_fds=True, + pass_fds=passfds, cwd=None, env=None, p2cread=-1, p2cwrite=-1, + c2pread=-1, c2pwrite=-1, errread=-1, errwrite=-1, + errpipe_read=errpipe_read, errpipe_write=errpipe_write, + restore_signals=False, call_setsid=False, pgid_to_set=-1, gid=None, + groups_list=None, uid=None, child_umask=-1, preexec_fn=None, + allow_vfork=subprocess._USE_VFORK) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 44e60d7c14954d..b4ae0280667361 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -75,6 +75,20 @@ static struct PyModuleDef _posixsubprocessmodule; +/*[clinic input] +module _posixsubprocess +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c62211df27cf7334]*/ + +/*[python input] +class pid_t_converter(CConverter): + type = 'pid_t' + format_unit = '" _Py_PARSE_PID "' +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=0c1d19f640d57e48]*/ + +#include "clinic/_posixsubprocess.c.h" + /* Convert ASCII to a positive int, no libc call. no overflow. -1 on error. */ static int _pos_int_from_ascii(const char *name) @@ -784,26 +798,76 @@ do_fork_exec(char *const exec_array[], return 0; /* Dead code to avoid a potential compiler warning. */ } +/*[clinic input] +_posixsubprocess.fork_exec as subprocess_fork_exec + args as process_args: object + executable_list: object + close_fds: bool + pass_fds as py_fds_to_keep: object(subclass_of='&PyTuple_Type') + cwd as cwd_obj: object + env as env_list: object + p2cread: int + p2cwrite: int + c2pread: int + c2pwrite: int + errread: int + errwrite: int + errpipe_read: int + errpipe_write: int + restore_signals: int + call_setsid: int + pgid_to_set: pid_t + gid as gid_object: object + groups_list: object + uid as uid_object: object + child_umask: int + preexec_fn: object + allow_vfork: bool + / + +Spawn a fresh new child process. + +Fork a child process, close parent file descriptors as appropriate in the +child and duplicate the few that are needed before calling exec() in the +child process. + +If close_fds is True, close file descriptors 3 and higher, except those listed +in the sorted tuple pass_fds. + +The preexec_fn, if supplied, will be called immediately before closing file +descriptors and exec. + +WARNING: preexec_fn is NOT SAFE if your application uses threads. + It may trigger infrequent, difficult to debug deadlocks. + +If an error occurs in the child process before the exec, it is +serialized and written to the errpipe_write fd per subprocess.py. + +Returns: the child process's PID. + +Raises: Only on an error in the parent process. +[clinic start generated code]*/ static PyObject * -subprocess_fork_exec(PyObject *module, PyObject *args) +subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, + PyObject *executable_list, int close_fds, + PyObject *py_fds_to_keep, PyObject *cwd_obj, + PyObject *env_list, int p2cread, int p2cwrite, + int c2pread, int c2pwrite, int errread, + int errwrite, int errpipe_read, int errpipe_write, + int restore_signals, int call_setsid, + pid_t pgid_to_set, PyObject *gid_object, + PyObject *groups_list, PyObject *uid_object, + int child_umask, PyObject *preexec_fn, + int allow_vfork) +/*[clinic end generated code: output=7c8ff5a6dc92af1b input=da74d2ddbd5de762]*/ { - PyObject *gc_module = NULL; - PyObject *executable_list, *py_fds_to_keep; - PyObject *env_list, *preexec_fn; - PyObject *process_args, *converted_args = NULL, *fast_args = NULL; + PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; - PyObject *groups_list; - PyObject *uid_object, *gid_object; - int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite; - int errpipe_read, errpipe_write, close_fds, restore_signals; - int call_setsid; - pid_t pgid_to_set = -1; int call_setgid = 0, call_setgroups = 0, call_setuid = 0; uid_t uid; gid_t gid, *groups = NULL; - int child_umask; - PyObject *cwd_obj, *cwd_obj2 = NULL; + PyObject *cwd_obj2 = NULL; const char *cwd; pid_t pid = -1; int need_to_reenable_gc = 0; @@ -811,19 +875,6 @@ subprocess_fork_exec(PyObject *module, PyObject *args) Py_ssize_t arg_num, num_groups = 0; int need_after_fork = 0; int saved_errno = 0; - int allow_vfork; - - if (!PyArg_ParseTuple( - args, "OOpO!OOiiiiiiiiii" _Py_PARSE_PID "OOOiOp:fork_exec", - &process_args, &executable_list, - &close_fds, &PyTuple_Type, &py_fds_to_keep, - &cwd_obj, &env_list, - &p2cread, &p2cwrite, &c2pread, &c2pwrite, - &errread, &errwrite, &errpipe_read, &errpipe_write, - &restore_signals, &call_setsid, &pgid_to_set, - &gid_object, &groups_list, &uid_object, &child_umask, - &preexec_fn, &allow_vfork)) - return NULL; if ((preexec_fn != Py_None) && (PyInterpreterState_Get() != PyInterpreterState_Main())) { @@ -1079,47 +1130,17 @@ subprocess_fork_exec(PyObject *module, PyObject *args) if (need_to_reenable_gc) { PyGC_Enable(); } - Py_XDECREF(gc_module); return pid == -1 ? NULL : PyLong_FromPid(pid); } - -PyDoc_STRVAR(subprocess_fork_exec_doc, -"fork_exec(args, executable_list, close_fds, pass_fds, cwd, env,\n\ - p2cread, p2cwrite, c2pread, c2pwrite,\n\ - errread, errwrite, errpipe_read, errpipe_write,\n\ - restore_signals, call_setsid, pgid_to_set,\n\ - gid, groups_list, uid,\n\ - preexec_fn)\n\ -\n\ -Forks a child process, closes parent file descriptors as appropriate in the\n\ -child and dups the few that are needed before calling exec() in the child\n\ -process.\n\ -\n\ -If close_fds is true, close file descriptors 3 and higher, except those listed\n\ -in the sorted tuple pass_fds.\n\ -\n\ -The preexec_fn, if supplied, will be called immediately before closing file\n\ -descriptors and exec.\n\ -WARNING: preexec_fn is NOT SAFE if your application uses threads.\n\ - It may trigger infrequent, difficult to debug deadlocks.\n\ -\n\ -If an error occurs in the child process before the exec, it is\n\ -serialized and written to the errpipe_write fd per subprocess.py.\n\ -\n\ -Returns: the child process's PID.\n\ -\n\ -Raises: Only on an error in the parent process.\n\ -"); - /* module level code ********************************************************/ PyDoc_STRVAR(module_doc, "A POSIX helper for the subprocess module."); static PyMethodDef module_methods[] = { - {"fork_exec", subprocess_fork_exec, METH_VARARGS, subprocess_fork_exec_doc}, + _POSIXSUBPROCESS_FORK_EXEC_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h new file mode 100644 index 00000000000000..88f83c2a03b945 --- /dev/null +++ b/Modules/clinic/_posixsubprocess.c.h @@ -0,0 +1,88 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(subprocess_fork_exec__doc__, +"fork_exec($module, args, executable_list, close_fds, pass_fds, cwd,\n" +" env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" +" errpipe_read, errpipe_write, restore_signals, call_setsid,\n" +" pgid_to_set, gid, groups_list, uid, child_umask, preexec_fn,\n" +" allow_vfork, /)\n" +"--\n" +"\n" +"Spawn a fresh new child process.\n" +"\n" +"Fork a child process, close parent file descriptors as appropriate in the\n" +"child and duplicate the few that are needed before calling exec() in the\n" +"child process.\n" +"\n" +"If close_fds is True, close file descriptors 3 and higher, except those listed\n" +"in the sorted tuple pass_fds.\n" +"\n" +"The preexec_fn, if supplied, will be called immediately before closing file\n" +"descriptors and exec.\n" +"\n" +"WARNING: preexec_fn is NOT SAFE if your application uses threads.\n" +" It may trigger infrequent, difficult to debug deadlocks.\n" +"\n" +"If an error occurs in the child process before the exec, it is\n" +"serialized and written to the errpipe_write fd per subprocess.py.\n" +"\n" +"Returns: the child process\'s PID.\n" +"\n" +"Raises: Only on an error in the parent process."); + +#define SUBPROCESS_FORK_EXEC_METHODDEF \ + {"fork_exec", (PyCFunction)(void(*)(void))subprocess_fork_exec, METH_FASTCALL, subprocess_fork_exec__doc__}, + +static PyObject * +subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, + PyObject *executable_list, int close_fds, + PyObject *py_fds_to_keep, PyObject *cwd_obj, + PyObject *env_list, int p2cread, int p2cwrite, + int c2pread, int c2pwrite, int errread, + int errwrite, int errpipe_read, int errpipe_write, + int restore_signals, int call_setsid, + pid_t pgid_to_set, PyObject *gid_object, + PyObject *groups_list, PyObject *uid_object, + int child_umask, PyObject *preexec_fn, + int allow_vfork); + +static PyObject * +subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *process_args; + PyObject *executable_list; + int close_fds; + PyObject *py_fds_to_keep; + PyObject *cwd_obj; + PyObject *env_list; + int p2cread; + int p2cwrite; + int c2pread; + int c2pwrite; + int errread; + int errwrite; + int errpipe_read; + int errpipe_write; + int restore_signals; + int call_setsid; + pid_t pgid_to_set; + PyObject *gid_object; + PyObject *groups_list; + PyObject *uid_object; + int child_umask; + PyObject *preexec_fn; + int allow_vfork; + + if (!_PyArg_ParseStack(args, nargs, "OOpO!OOiiiiiiiiii" _Py_PARSE_PID "OOOiOp:fork_exec", + &process_args, &executable_list, &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, &restore_signals, &call_setsid, &pgid_to_set, &gid_object, &groups_list, &uid_object, &child_umask, &preexec_fn, &allow_vfork)) { + goto exit; + } + return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, groups_list, uid_object, child_umask, preexec_fn, allow_vfork); + +exit: + return return_value; +} +/*[clinic end generated code: output=d0909dc97cac8f01 input=a9049054013a1b77]*/ From 514a377d658161b8ddaeac12449b752006bd06a1 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 2 Jul 2022 21:47:00 +0300 Subject: [PATCH 02/20] Make flag parameters boolean --- Modules/_posixsubprocess.c | 6 +++--- Modules/clinic/_posixsubprocess.c.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b4ae0280667361..83ff4b1082223b 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -814,8 +814,8 @@ _posixsubprocess.fork_exec as subprocess_fork_exec errwrite: int errpipe_read: int errpipe_write: int - restore_signals: int - call_setsid: int + restore_signals: bool + call_setsid: bool pgid_to_set: pid_t gid as gid_object: object groups_list: object @@ -860,7 +860,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *groups_list, PyObject *uid_object, int child_umask, PyObject *preexec_fn, int allow_vfork) -/*[clinic end generated code: output=7c8ff5a6dc92af1b input=da74d2ddbd5de762]*/ +/*[clinic end generated code: output=7c8ff5a6dc92af1b input=c59d1152ecdffcf9]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index 88f83c2a03b945..40ccc87d812921 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -76,7 +76,7 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *preexec_fn; int allow_vfork; - if (!_PyArg_ParseStack(args, nargs, "OOpO!OOiiiiiiiiii" _Py_PARSE_PID "OOOiOp:fork_exec", + if (!_PyArg_ParseStack(args, nargs, "OOpO!OOiiiiiiiipp" _Py_PARSE_PID "OOOiOp:fork_exec", &process_args, &executable_list, &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, &restore_signals, &call_setsid, &pgid_to_set, &gid_object, &groups_list, &uid_object, &child_umask, &preexec_fn, &allow_vfork)) { goto exit; } @@ -85,4 +85,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d0909dc97cac8f01 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=847823a9cb220990 input=a9049054013a1b77]*/ From f6ffbb786b7e4d872e1185ebc2d35319018847d6 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 2 Jul 2022 21:48:14 +0300 Subject: [PATCH 03/20] Regenerate clinic 3.10 files using clinic 3.12 --- Modules/clinic/_posixsubprocess.c.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index 40ccc87d812921..e597c853c038a1 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -33,7 +33,7 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "Raises: Only on an error in the parent process."); #define SUBPROCESS_FORK_EXEC_METHODDEF \ - {"fork_exec", (PyCFunction)(void(*)(void))subprocess_fork_exec, METH_FASTCALL, subprocess_fork_exec__doc__}, + {"fork_exec", _PyCFunction_CAST(subprocess_fork_exec), METH_FASTCALL, subprocess_fork_exec__doc__}, static PyObject * subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, @@ -85,4 +85,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=847823a9cb220990 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5ad4c6bc39662d25 input=a9049054013a1b77]*/ From bed7d603879027d4843f8becb2cfedb4573a2fa0 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 3 Jul 2022 01:13:23 +0300 Subject: [PATCH 04/20] Fix forced arg format --- Modules/_posixsubprocess.c | 10 +++- Modules/clinic/_posixsubprocess.c.h | 74 +++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 83ff4b1082223b..58a4b2dc83bd2f 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -84,8 +84,16 @@ module _posixsubprocess class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' + + def parse_arg(self, argname, displayname): + return """ + {paramname} = PyLong_AsPid({argname}); + if ({paramname} == -1 && PyErr_Occurred()) {{{{ + goto exit; + }}}} + """.format(argname=argname, paramname=self.parser_name) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=0c1d19f640d57e48]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=5af1c116d56cbb5a]*/ #include "clinic/_posixsubprocess.c.h" diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index e597c853c038a1..83ced5d421f6f4 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -76,8 +76,76 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *preexec_fn; int allow_vfork; - if (!_PyArg_ParseStack(args, nargs, "OOpO!OOiiiiiiiipp" _Py_PARSE_PID "OOOiOp:fork_exec", - &process_args, &executable_list, &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, &restore_signals, &call_setsid, &pgid_to_set, &gid_object, &groups_list, &uid_object, &child_umask, &preexec_fn, &allow_vfork)) { + if (!_PyArg_CheckPositional("fork_exec", nargs, 23, 23)) { + goto exit; + } + process_args = args[0]; + executable_list = args[1]; + close_fds = PyObject_IsTrue(args[2]); + if (close_fds < 0) { + goto exit; + } + if (!PyTuple_Check(args[3])) { + _PyArg_BadArgument("fork_exec", "argument 4", "tuple", args[3]); + goto exit; + } + py_fds_to_keep = args[3]; + cwd_obj = args[4]; + env_list = args[5]; + p2cread = _PyLong_AsInt(args[6]); + if (p2cread == -1 && PyErr_Occurred()) { + goto exit; + } + p2cwrite = _PyLong_AsInt(args[7]); + if (p2cwrite == -1 && PyErr_Occurred()) { + goto exit; + } + c2pread = _PyLong_AsInt(args[8]); + if (c2pread == -1 && PyErr_Occurred()) { + goto exit; + } + c2pwrite = _PyLong_AsInt(args[9]); + if (c2pwrite == -1 && PyErr_Occurred()) { + goto exit; + } + errread = _PyLong_AsInt(args[10]); + if (errread == -1 && PyErr_Occurred()) { + goto exit; + } + errwrite = _PyLong_AsInt(args[11]); + if (errwrite == -1 && PyErr_Occurred()) { + goto exit; + } + errpipe_read = _PyLong_AsInt(args[12]); + if (errpipe_read == -1 && PyErr_Occurred()) { + goto exit; + } + errpipe_write = _PyLong_AsInt(args[13]); + if (errpipe_write == -1 && PyErr_Occurred()) { + goto exit; + } + restore_signals = PyObject_IsTrue(args[14]); + if (restore_signals < 0) { + goto exit; + } + call_setsid = PyObject_IsTrue(args[15]); + if (call_setsid < 0) { + goto exit; + } + pgid_to_set = PyLong_AsPid(args[16]); + if (pgid_to_set == -1 && PyErr_Occurred()) { + goto exit; + } + gid_object = args[17]; + groups_list = args[18]; + uid_object = args[19]; + child_umask = _PyLong_AsInt(args[20]); + if (child_umask == -1 && PyErr_Occurred()) { + goto exit; + } + preexec_fn = args[21]; + allow_vfork = PyObject_IsTrue(args[22]); + if (allow_vfork < 0) { goto exit; } return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, groups_list, uid_object, child_umask, preexec_fn, allow_vfork); @@ -85,4 +153,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5ad4c6bc39662d25 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b976621b8861610b input=a9049054013a1b77]*/ From 603be009e22a9a85ccfa6118b275e794ff9c434e Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 3 Jul 2022 11:49:56 +0300 Subject: [PATCH 05/20] Fixed a non-existent name left after renaming --- Modules/_posixsubprocess.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 58a4b2dc83bd2f..813de236c9135c 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1148,7 +1148,7 @@ PyDoc_STRVAR(module_doc, "A POSIX helper for the subprocess module."); static PyMethodDef module_methods[] = { - _POSIXSUBPROCESS_FORK_EXEC_METHODDEF + SUBPROCESS_FORK_EXEC_METHODDEF {NULL, NULL} /* sentinel */ }; From a0646e688536980579fa0a909a707d5735e8484a Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 3 Jul 2022 12:11:56 +0300 Subject: [PATCH 06/20] Revert named arguments in multiprocessing/util.py --- Lib/multiprocessing/util.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index cb7b3cd81a079e..6ee0d33e88a060 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -451,13 +451,10 @@ def spawnv_passfds(path, args, passfds): errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( - args, executable_list=[path], close_fds=True, - pass_fds=passfds, cwd=None, env=None, p2cread=-1, p2cwrite=-1, - c2pread=-1, c2pwrite=-1, errread=-1, errwrite=-1, - errpipe_read=errpipe_read, errpipe_write=errpipe_write, - restore_signals=False, call_setsid=False, pgid_to_set=-1, gid=None, - groups_list=None, uid=None, child_umask=-1, preexec_fn=None, - allow_vfork=subprocess._USE_VFORK) + args, [path], True, passfds, None, None, + -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, + False, False, -1, None, None, None, -1, None, + subprocess._USE_VFORK) finally: os.close(errpipe_read) os.close(errpipe_write) From 86edd0e8dbf5f6e353990b6772620c34f48336b9 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 3 Jul 2022 23:13:39 +0300 Subject: [PATCH 07/20] Add a NEWS entry --- .../next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst diff --git a/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst b/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst new file mode 100644 index 00000000000000..7719b74b8e5ef1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-03-23-13-28.gh-issue-94518.511Tbh.rst @@ -0,0 +1 @@ +Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. From fe48164a1a0f81b515291d4da55eec250b262c72 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 5 Jul 2022 06:22:56 +0300 Subject: [PATCH 08/20] Move uninitialized variables closer to their initialization --- Modules/_posixsubprocess.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 813de236c9135c..c6eb4202235981 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -873,14 +873,12 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; int call_setgid = 0, call_setgroups = 0, call_setuid = 0; - uid_t uid; - gid_t gid, *groups = NULL; + gid_t *groups = NULL; PyObject *cwd_obj2 = NULL; - const char *cwd; - pid_t pid = -1; + const char *cwd = NULL; int need_to_reenable_gc = 0; - char *const *exec_array, *const *argv = NULL, *const *envp = NULL; - Py_ssize_t arg_num, num_groups = 0; + char *const *argv = NULL, *const *envp = NULL; + Py_ssize_t num_groups = 0; int need_after_fork = 0; int saved_errno = 0; @@ -913,7 +911,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, need_to_reenable_gc = PyGC_Disable(); } - exec_array = _PySequence_BytesToCharpArray(executable_list); + char *const *exec_array = _PySequence_BytesToCharpArray(executable_list); if (!exec_array) goto cleanup; @@ -931,7 +929,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, converted_args = PyTuple_New(num_args); if (converted_args == NULL) goto cleanup; - for (arg_num = 0; arg_num < num_args; ++arg_num) { + for (Py_ssize_t arg_num = 0; arg_num < num_args; ++arg_num) { PyObject *borrowed_arg, *converted_arg; if (PySequence_Fast_GET_SIZE(fast_args) != num_args) { PyErr_SetString(PyExc_RuntimeError, "args changed during iteration"); @@ -960,8 +958,6 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, if (PyUnicode_FSConverter(cwd_obj, &cwd_obj2) == 0) goto cleanup; cwd = PyBytes_AsString(cwd_obj2); - } else { - cwd = NULL; } if (groups_list != Py_None) { @@ -1001,6 +997,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_DECREF(elem); goto cleanup; } else { + gid_t gid; if (!_Py_Gid_Converter(elem, &gid)) { Py_DECREF(elem); PyErr_SetString(PyExc_ValueError, "invalid group id"); @@ -1031,6 +1028,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETREUID */ } + uid_t uid; if (uid_object != Py_None) { #ifdef HAVE_SETREUID if (!_Py_Uid_Converter(uid_object, &uid)) @@ -1080,6 +1078,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, } #endif + pid_t pid; pid = do_fork_exec(exec_array, argv, envp, cwd, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, From 90bb494c0c9b0de36e85108b204390e3387507d1 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 5 Jul 2022 06:46:28 +0300 Subject: [PATCH 09/20] Mark do_fork_exec as C11 _Noreturn and remove `return 0` hack --- Modules/_posixsubprocess.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index c6eb4202235981..9a4aebda268632 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1015,6 +1015,8 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETGROUPS */ } + /* if gid is left uninitialized, call_setgid == 0 so no read access. */ + gid_t gid; if (gid_object != Py_None) { #ifdef HAVE_SETREGID if (!_Py_Gid_Converter(gid_object, &gid)) @@ -1028,6 +1030,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETREUID */ } + /* if uid is left uninitialized, call_setuid == 0 so no read access. */ uid_t uid; if (uid_object != Py_None) { #ifdef HAVE_SETREUID From d0a586c5c5177daeccf3c4eb8fede654e0fa0b8c Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 5 Jul 2022 12:56:24 +0300 Subject: [PATCH 10/20] Address the review on undefined gid/uid --- Modules/_posixsubprocess.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 9a4aebda268632..468f4346aca2d3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -498,6 +498,9 @@ reset_signal_handlers(const sigset_t *child_sigmask) #endif /* VFORK_USABLE */ +#define RESERVED_UID (uid_t)-1 +#define RESERVED_GID (gid_t)-1 + /* * This function is code executed in the child process immediately after * (v)fork to set things up and call exec(). @@ -646,12 +649,12 @@ child_exec(char *const exec_array[], #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID - if (call_setgid) + if (gid == RESERVED_GID) POSIX_CALL(setregid(gid, gid)); #endif /* HAVE_SETREGID */ #ifdef HAVE_SETREUID - if (call_setuid) + if (uid == RESERVED_UID) POSIX_CALL(setreuid(uid, uid)); #endif /* HAVE_SETREUID */ @@ -1015,8 +1018,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETGROUPS */ } - /* if gid is left uninitialized, call_setgid == 0 so no read access. */ - gid_t gid; + gid_t gid = RESERVED_GID; if (gid_object != Py_None) { #ifdef HAVE_SETREGID if (!_Py_Gid_Converter(gid_object, &gid)) @@ -1030,8 +1032,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETREUID */ } - /* if uid is left uninitialized, call_setuid == 0 so no read access. */ - uid_t uid; + uid_t uid = RESERVED_UID; if (uid_object != Py_None) { #ifdef HAVE_SETREUID if (!_Py_Uid_Converter(uid_object, &uid)) From df6bb7257db91fe2e514db34d599eaecf2878da8 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 6 Jul 2022 22:02:32 +0300 Subject: [PATCH 11/20] Fix an unitialized PID --- Modules/_posixsubprocess.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 468f4346aca2d3..6b45d15855d5da 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -498,8 +498,9 @@ reset_signal_handlers(const sigset_t *child_sigmask) #endif /* VFORK_USABLE */ -#define RESERVED_UID (uid_t)-1 #define RESERVED_GID (gid_t)-1 +#define RESERVED_PID (pid_t)-1 +#define RESERVED_UID (uid_t)-1 /* * This function is code executed in the child process immediately after @@ -758,7 +759,7 @@ do_fork_exec(char *const exec_array[], PyObject *preexec_fn_args_tuple) { - pid_t pid; + pid_t pid = 0; #ifdef VFORK_USABLE if (child_sigmask) { @@ -769,7 +770,7 @@ do_fork_exec(char *const exec_array[], assert(preexec_fn == Py_None); pid = vfork(); - if (pid == -1) { + if (pid == RESERVED_PID) { /* If vfork() fails, fall back to using fork(). When it isn't * allowed in a process by the kernel, vfork can return -1 * with errno EINVAL. https://bugs.python.org/issue47151. */ @@ -1092,7 +1093,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); /* Parent (original) process */ - if (pid == -1) { + if (pid == RESERVED_PID) { /* Capture errno for the exception. */ saved_errno = errno; } From c9275336521f20968a85de9076d5f1a6d7b25c35 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Thu, 7 Jul 2022 08:50:42 +0300 Subject: [PATCH 12/20] Remove confusing out-of-place variable definitions --- Modules/_posixsubprocess.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 6b45d15855d5da..fcb62897a7132f 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -966,9 +966,6 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, if (groups_list != Py_None) { #ifdef HAVE_SETGROUPS - Py_ssize_t i; - gid_t gid; - if (!PyList_Check(groups_list)) { PyErr_SetString(PyExc_TypeError, "setgroups argument must be a list"); @@ -990,7 +987,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, goto cleanup; } - for (i = 0; i < num_groups; i++) { + for (Py_ssize_t i = 0; i < num_groups; i++) { PyObject *elem; elem = PySequence_GetItem(groups_list, i); if (!elem) From 9cb12683028426008668dbe383f234097551d33b Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Thu, 7 Jul 2022 09:54:03 +0300 Subject: [PATCH 13/20] Fix fluke (?) "pid may be used uninitialized in this function" --- Modules/_posixsubprocess.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index fcb62897a7132f..25c15c57f80a44 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1080,8 +1080,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, } #endif - pid_t pid; - pid = do_fork_exec(exec_array, argv, envp, cwd, + pid_t pid = do_fork_exec(exec_array, argv, envp, cwd, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, close_fds, restore_signals, call_setsid, pgid_to_set, From 18e8821960879a8d3ceeb5d3c6ba2b07b38f454a Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Thu, 7 Jul 2022 10:38:34 +0300 Subject: [PATCH 14/20] Fix a minor grammar error --- Modules/_posixsubprocess.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 25c15c57f80a44..989c5ba6c5bbdc 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -650,12 +650,12 @@ child_exec(char *const exec_array[], #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID - if (gid == RESERVED_GID) + if (gid != RESERVED_GID) POSIX_CALL(setregid(gid, gid)); #endif /* HAVE_SETREGID */ #ifdef HAVE_SETREUID - if (uid == RESERVED_UID) + if (uid != RESERVED_UID) POSIX_CALL(setreuid(uid, uid)); #endif /* HAVE_SETREUID */ From c822f10e1279cf59e0c11d94c2cde03d00a66910 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 26 Jul 2022 07:19:17 +0300 Subject: [PATCH 15/20] Address Gregory's review --- Modules/_posixsubprocess.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 989c5ba6c5bbdc..0d4b04b2dcdaac 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -498,10 +498,6 @@ reset_signal_handlers(const sigset_t *child_sigmask) #endif /* VFORK_USABLE */ -#define RESERVED_GID (gid_t)-1 -#define RESERVED_PID (pid_t)-1 -#define RESERVED_UID (uid_t)-1 - /* * This function is code executed in the child process immediately after * (v)fork to set things up and call exec(). @@ -650,12 +646,12 @@ child_exec(char *const exec_array[], #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID - if (gid != RESERVED_GID) + if (gid != (gid_t)-1) POSIX_CALL(setregid(gid, gid)); #endif /* HAVE_SETREGID */ #ifdef HAVE_SETREUID - if (uid != RESERVED_UID) + if (uid != (uid_t)-1) POSIX_CALL(setreuid(uid, uid)); #endif /* HAVE_SETREUID */ @@ -770,7 +766,7 @@ do_fork_exec(char *const exec_array[], assert(preexec_fn == Py_None); pid = vfork(); - if (pid == RESERVED_PID) { + if (pid == (pid_t)-1) { /* If vfork() fails, fall back to using fork(). When it isn't * allowed in a process by the kernel, vfork can return -1 * with errno EINVAL. https://bugs.python.org/issue47151. */ @@ -1016,7 +1012,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETGROUPS */ } - gid_t gid = RESERVED_GID; + gid_t gid = (gid_t)-1; if (gid_object != Py_None) { #ifdef HAVE_SETREGID if (!_Py_Gid_Converter(gid_object, &gid)) @@ -1030,7 +1026,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, #endif /* HAVE_SETREUID */ } - uid_t uid = RESERVED_UID; + uid_t uid = (uid_t)-1; if (uid_object != Py_None) { #ifdef HAVE_SETREUID if (!_Py_Uid_Converter(uid_object, &uid)) @@ -1089,7 +1085,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); /* Parent (original) process */ - if (pid == RESERVED_PID) { + if (pid == (pid_t)-1) { /* Capture errno for the exception. */ saved_errno = errno; } From 2535f5ed76d29c7235d0a3763ba679e95fb88e31 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 15 Jan 2023 11:10:06 +0400 Subject: [PATCH 16/20] Update a clinic file --- Modules/clinic/_posixsubprocess.c.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index 83ced5d421f6f4..dfaa17ab0b0497 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -2,6 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + PyDoc_STRVAR(subprocess_fork_exec__doc__, "fork_exec($module, args, executable_list, close_fds, pass_fds, cwd,\n" " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" @@ -153,4 +159,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=b976621b8861610b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=772b8fdb9b982c33 input=a9049054013a1b77]*/ From 6822d1040a5999a88b8154c41f979abfe2e0e6c2 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 15 Jan 2023 12:19:39 +0400 Subject: [PATCH 17/20] Remove a redundant initialization already covered in all `if` branches below --- Modules/_posixsubprocess.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index dbfdeb74f82847..ac1fc915eb0bbd 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -755,7 +755,7 @@ do_fork_exec(char *const exec_array[], PyObject *preexec_fn_args_tuple) { - pid_t pid = 0; + pid_t pid; #ifdef VFORK_USABLE if (child_sigmask) { From e09851fe6de37de342194e4cfe24e16f786410e9 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Thu, 26 Jan 2023 12:24:17 +0400 Subject: [PATCH 18/20] Bring groups -> extra_groups from the main --- Modules/_posixsubprocess.c | 60 ++++++++++++++--------------- Modules/clinic/_posixsubprocess.c.h | 16 ++++---- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index ac1fc915eb0bbd..6b7aaa8c403e0b 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -541,7 +541,7 @@ child_exec(char *const exec_array[], int close_fds, int restore_signals, int call_setsid, pid_t pgid_to_set, gid_t gid, - Py_ssize_t groups_size, const gid_t *groups, + Py_ssize_t extra_group_size, const gid_t *extra_groups, uid_t uid, int child_umask, const void *child_sigmask, PyObject *py_fds_to_keep, @@ -641,8 +641,8 @@ child_exec(char *const exec_array[], #endif #ifdef HAVE_SETGROUPS - if (groups_size > 0) - POSIX_CALL(setgroups(groups_size, groups)); + if (extra_group_size > 0) + POSIX_CALL(setgroups(extra_group_size, extra_groups)); #endif /* HAVE_SETGROUPS */ #ifdef HAVE_SETREGID @@ -747,7 +747,7 @@ do_fork_exec(char *const exec_array[], int close_fds, int restore_signals, int call_setsid, pid_t pgid_to_set, gid_t gid, - Py_ssize_t groups_size, const gid_t *groups, + Py_ssize_t extra_group_size, const gid_t *extra_groups, uid_t uid, int child_umask, const void *child_sigmask, PyObject *py_fds_to_keep, @@ -762,7 +762,7 @@ do_fork_exec(char *const exec_array[], /* These are checked by our caller; verify them in debug builds. */ assert(uid == (uid_t)-1); assert(gid == (gid_t)-1); - assert(groups_size < 0); + assert(extra_group_size < 0); assert(preexec_fn == Py_None); pid = vfork(); @@ -799,7 +799,7 @@ do_fork_exec(char *const exec_array[], p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, close_fds, restore_signals, call_setsid, pgid_to_set, - gid, groups_size, groups, + gid, extra_group_size, extra_groups, uid, child_umask, child_sigmask, py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); _exit(255); @@ -826,7 +826,7 @@ _posixsubprocess.fork_exec as subprocess_fork_exec call_setsid: bool pgid_to_set: pid_t gid as gid_object: object - groups_list: object + extra_groups as extra_groups_packed: object uid as uid_object: object child_umask: int preexec_fn: object @@ -865,19 +865,19 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, int errwrite, int errpipe_read, int errpipe_write, int restore_signals, int call_setsid, pid_t pgid_to_set, PyObject *gid_object, - PyObject *groups_list, PyObject *uid_object, - int child_umask, PyObject *preexec_fn, - int allow_vfork) -/*[clinic end generated code: output=7c8ff5a6dc92af1b input=c59d1152ecdffcf9]*/ + PyObject *extra_groups_packed, + PyObject *uid_object, int child_umask, + PyObject *preexec_fn, int allow_vfork) +/*[clinic end generated code: output=7ee4f6ee5cf22b5b input=51757287ef266ffa]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; - gid_t *groups = NULL; + gid_t *extra_groups = NULL; PyObject *cwd_obj2 = NULL; const char *cwd = NULL; int need_to_reenable_gc = 0; char *const *argv = NULL, *const *envp = NULL; - Py_ssize_t num_groups = 0; + Py_ssize_t extra_group_size = 0; int need_after_fork = 0; int saved_errno = 0; @@ -951,41 +951,41 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, cwd = PyBytes_AsString(cwd_obj2); } - if (groups_list != Py_None) { + if (extra_groups_packed != Py_None) { #ifdef HAVE_SETGROUPS - if (!PyList_Check(groups_list)) { + if (!PyList_Check(extra_groups_packed)) { PyErr_SetString(PyExc_TypeError, "setgroups argument must be a list"); goto cleanup; } - num_groups = PySequence_Size(groups_list); + extra_group_size = PySequence_Size(extra_groups_packed); - if (num_groups < 0) + if (extra_group_size < 0) goto cleanup; - if (num_groups > MAX_GROUPS) { - PyErr_SetString(PyExc_ValueError, "too many groups"); + if (extra_group_size > MAX_GROUPS) { + PyErr_SetString(PyExc_ValueError, "too many extra_groups"); goto cleanup; } - /* Deliberately keep groups == NULL for num_groups == 0 */ - if (num_groups > 0) { - groups = PyMem_RawMalloc(num_groups * sizeof(gid_t)); - if (groups == NULL) { + /* Deliberately keep extra_groups == NULL for extra_group_size == 0 */ + if (extra_group_size > 0) { + extra_groups = PyMem_RawMalloc(extra_group_size * sizeof(gid_t)); + if (extra_groups == NULL) { PyErr_SetString(PyExc_MemoryError, "failed to allocate memory for group list"); goto cleanup; } } - for (Py_ssize_t i = 0; i < num_groups; i++) { + for (Py_ssize_t i = 0; i < extra_group_size; i++) { PyObject *elem; - elem = PySequence_GetItem(groups_list, i); + elem = PySequence_GetItem(extra_groups_packed, i); if (!elem) goto cleanup; if (!PyLong_Check(elem)) { PyErr_SetString(PyExc_TypeError, - "groups must be integers"); + "extra_groups must be integers"); Py_DECREF(elem); goto cleanup; } else { @@ -995,7 +995,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyErr_SetString(PyExc_ValueError, "invalid group id"); goto cleanup; } - groups[i] = gid; + extra_groups[i] = gid; } Py_DECREF(elem); } @@ -1047,7 +1047,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, /* Use vfork() only if it's safe. See the comment above child_exec(). */ sigset_t old_sigs; if (preexec_fn == Py_None && allow_vfork && - uid == (uid_t)-1 && gid == (gid_t)-1 && num_groups < 0) { + uid == (uid_t)-1 && gid == (gid_t)-1 && extra_group_size < 0) { /* Block all signals to ensure that no signal handlers are run in the * child process while it shares memory with us. Note that signals * used internally by C libraries won't be blocked by @@ -1070,7 +1070,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, close_fds, restore_signals, call_setsid, pgid_to_set, - gid, num_groups, groups, + gid, extra_group_size, extra_groups, uid, child_umask, old_sigmask, py_fds_to_keep, preexec_fn, preexec_fn_args_tuple); @@ -1110,7 +1110,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, } Py_XDECREF(preexec_fn_args_tuple); - PyMem_RawFree(groups); + PyMem_RawFree(extra_groups); Py_XDECREF(cwd_obj2); if (envp) _Py_FreeCharPArray(envp); diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index dfaa17ab0b0497..f08878cf668908 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -12,7 +12,7 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "fork_exec($module, args, executable_list, close_fds, pass_fds, cwd,\n" " env, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite,\n" " errpipe_read, errpipe_write, restore_signals, call_setsid,\n" -" pgid_to_set, gid, groups_list, uid, child_umask, preexec_fn,\n" +" pgid_to_set, gid, extra_groups, uid, child_umask, preexec_fn,\n" " allow_vfork, /)\n" "--\n" "\n" @@ -50,9 +50,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, int errwrite, int errpipe_read, int errpipe_write, int restore_signals, int call_setsid, pid_t pgid_to_set, PyObject *gid_object, - PyObject *groups_list, PyObject *uid_object, - int child_umask, PyObject *preexec_fn, - int allow_vfork); + PyObject *extra_groups_packed, + PyObject *uid_object, int child_umask, + PyObject *preexec_fn, int allow_vfork); static PyObject * subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -76,7 +76,7 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int call_setsid; pid_t pgid_to_set; PyObject *gid_object; - PyObject *groups_list; + PyObject *extra_groups_packed; PyObject *uid_object; int child_umask; PyObject *preexec_fn; @@ -143,7 +143,7 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } gid_object = args[17]; - groups_list = args[18]; + extra_groups_packed = args[18]; uid_object = args[19]; child_umask = _PyLong_AsInt(args[20]); if (child_umask == -1 && PyErr_Occurred()) { @@ -154,9 +154,9 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (allow_vfork < 0) { goto exit; } - return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, groups_list, uid_object, child_umask, preexec_fn, allow_vfork); + return_value = subprocess_fork_exec_impl(module, process_args, executable_list, close_fds, py_fds_to_keep, cwd_obj, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, call_setsid, pgid_to_set, gid_object, extra_groups_packed, uid_object, child_umask, preexec_fn, allow_vfork); exit: return return_value; } -/*[clinic end generated code: output=772b8fdb9b982c33 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=46d71e86845c93d7 input=a9049054013a1b77]*/ From 7251169ab52314e66c6b0fdc4320ede73e192290 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 12 Apr 2023 17:38:39 +0400 Subject: [PATCH 19/20] Fix reading from uninitialized `pid` --- Modules/_posixsubprocess.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 6b7aaa8c403e0b..3af39cee4f4772 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -873,6 +873,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; gid_t *extra_groups = NULL; + pid_t pid = -1; PyObject *cwd_obj2 = NULL; const char *cwd = NULL; int need_to_reenable_gc = 0; @@ -1066,7 +1067,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, } #endif - pid_t pid = do_fork_exec(exec_array, argv, envp, cwd, + pid = do_fork_exec(exec_array, argv, envp, cwd, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, close_fds, restore_signals, call_setsid, pgid_to_set, From c7e1a5e2b34b2ae76eb8bb93ea45bd3ea18017de Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 15 Apr 2023 15:25:11 +0400 Subject: [PATCH 20/20] Minimize the diff --- Modules/_posixsubprocess.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 3af39cee4f4772..f5bce8cd7628ad 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -873,9 +873,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; gid_t *extra_groups = NULL; - pid_t pid = -1; PyObject *cwd_obj2 = NULL; const char *cwd = NULL; + pid_t pid = -1; int need_to_reenable_gc = 0; char *const *argv = NULL, *const *envp = NULL; Py_ssize_t extra_group_size = 0;