From 919efccc188c0175056053164a70cff9d6fd3ef6 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:17 +0200 Subject: [PATCH 01/45] Add support for ARM64 to the pythonw tool --- Mac/Tools/pythonw.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index c8bd3ba8d68c15..8d65da79517420 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) #elif defined(__ppc__) cpu_types[0] = CPU_TYPE_POWERPC; + #elif defined(__i386__) cpu_types[0] = CPU_TYPE_X86; + +#elif defined(__arm64__) + cpu_types[0] = CPU_TYPE_ARM64; + #else # error "Unknown CPU" + #endif if (posix_spawnattr_setbinpref_np(spawnattr, count, From 69c39f36f6efe9b13b2dc06892133ac31e6ca7bc Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:43 +0200 Subject: [PATCH 02/45] Add support for "universal2" as a fat binary target on macOS --- Lib/_osx_support.py | 2 ++ configure | 20 ++++++++++++++------ configure.ac | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed5bd..8ed1eeac8525da 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -481,6 +481,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/configure b/configure index 5024860ca4395a..f811d2c23e4a7a 100755 --- a/configure +++ b/configure @@ -1509,8 +1509,8 @@ Optional Packages: specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", - "intel-64", or "all") see Mac/README.rst + ("universal2", "32-bit", "64-bit", "3-way", "intel", + "intel-32", "intel-64", or "all") see Mac/README.rst --with-framework-name=FRAMEWORK specify the name for the python framework on macOS only valid when --enable-framework is set. see @@ -6945,7 +6945,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7472,6 +7472,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7493,7 +7498,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9322,7 +9327,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9332,9 +9337,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac diff --git a/configure.ac b/configure.ac index 5a3e340aa3e72b..d2044d0520e679 100644 --- a/configure.ac +++ b/configure.ac @@ -218,7 +218,7 @@ AC_ARG_WITH(universal-archs, AS_HELP_STRING([--with-universal-archs=ARCH], [specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") + ("universal2", "32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") see Mac/README.rst]), [ UNIVERSAL_ARCHS="$withval" @@ -1578,7 +1578,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1879,6 +1879,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1900,7 +1905,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2467,7 +2472,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2477,9 +2482,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac From 3940c867b6394d8950bfff725133e8a8241081b0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 15:47:44 +0200 Subject: [PATCH 03/45] Tweak ctypes.macholib.dyld to work with the shared library cache. --- Lib/ctypes/macholib/dyld.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765a3e..9ab9e79ab816c4 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -122,7 +122,19 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): - if os.path.isfile(path): + + # on macOS 11 system libraries are in a shared library + # cache, not in the regular place in the filesystem. + # There still are symlinks (libz.dylib -> libz.1.dylib), + # but the target of the symlink no longer is there. + # + # There shouldn't be 3th-party libraries in these + # system locations + if path.startswith("/System/") and os.path.islink(path): + return path + elif path.startswith("/usr/lib/") and os.path.islink(path): + return path + elif os.path.isfile(path): return path raise ValueError("dylib %s could not be found" % (name,)) From deda5f0c830517969c619821f5d932495d7d27ef Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:11:40 +0200 Subject: [PATCH 04/45] Silence compile time warning We should consider just dropping support for macOS 10.4 here to simplify the code. --- Mac/Tools/pythonw.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index 8d65da79517420..78813e818e7dac 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -223,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); From ea3c200890e1dc606745f6a5308725018595d001 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:12:37 +0200 Subject: [PATCH 05/45] macOS/arm64 support, based on GH-21249 This is support for ctypes on macOS/arm64 based on PR 21249 by Lawrence D'Anna (Apple). Changes: - changed __builtin_available tests from 11.0 to 10.15 - added test to setup.py for ffi_closure_alloc and use that in malloc_closure.c - Minor change in the code path for ffi_prep_closure_var (coding style change) --- Lib/test/test_bytes.py | 1 + Lib/test/test_unicode.py | 2 + Modules/_ctypes/callbacks.c | 39 +++++++++--- Modules/_ctypes/callproc.c | 72 ++++++++++++++++++---- Modules/_ctypes/ctypes.h | 8 +++ Modules/_ctypes/malloc_closure.c | 15 ++++- setup.py | 102 ++++++++++++++++--------------- 7 files changed, 168 insertions(+), 71 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 61b4b9162ccc54..e26ea4704bf5c7 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1035,6 +1035,7 @@ def test_from_format(self): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index d485bc7ede2b92..4cf60591d2dd4d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2513,11 +2513,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): import_helper.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 2abfa67cdc06bf..77d4855277ecec 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include + #include #ifdef MS_WIN32 #include @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else +#endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; #else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 261ae5ceb9e48a..ac407c93c240e4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include + #ifdef MS_WIN32 #include #include @@ -812,7 +814,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,14 +838,60 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1212,9 +1261,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc5ff9..3f20031d671a8a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb336958c6f..4f220e42ff3fcc 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/setup.py b/setup.py index 21a5a58981fc15..65d479fcb184c6 100644 --- a/setup.py +++ b/setup.py @@ -229,6 +229,13 @@ def macosx_sdk_specified(): macosx_sdk_root() return MACOS_SDK_SPECIFIED +def is_macosx_at_least(vers): + if MACOS: + dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if dep_target: + return tuple(map(int, dep_target.split('.'))) >= vers + return False + def is_macosx_sdk_path(path): """ @@ -239,6 +246,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -2100,43 +2114,18 @@ def detect_tkinter(self): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and + (is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2149,11 +2138,10 @@ def detect_ctypes(self): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') + #sources.append('_ctypes/darwin/dlfcn_simple.c') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2183,31 +2171,47 @@ def detect_ctypes(self): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') - - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + # XXX: The define should only be added when actually using the system + # version (and not a locally compiled one) + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") + if os.path.exists(ffi_in_sdk): + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 552bca86a2e7049d0483b779a7776245608d172c Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:23:50 +0200 Subject: [PATCH 06/45] Fix test for macOS before 10.10 --- Lib/distutils/tests/test_build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index f9e0d766d870e0..6bb009a86f41eb 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target): # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: From e0c23a1d14446079e5cbee0b1d1624f242eebbc5 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:25:35 +0200 Subject: [PATCH 07/45] ARM64 is a valid platform for macOS --- Lib/test/test_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5ad306e0ed5795..45ca08819fb36c 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -246,7 +246,7 @@ def test_mac_ver(self): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') From e637a77f3017d3149b90c25932805112b52d43c4 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 17:21:37 +0200 Subject: [PATCH 08/45] Unconditionally use uint32_t here. The preprocessor guard in the old version doesn't work, and isn't really needed (10.4 only supported 32-bit code where unsigned long is the same as uint32_t). --- Modules/getpath.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index f7a6dd40443054..44453f29df703a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -923,11 +923,7 @@ static PyStatus calculate_program_macos(wchar_t **abs_path_p) { char execpath[MAXPATHLEN + 1]; -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#else - unsigned long nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#endif /* On Mac OS X, if a script uses an interpreter of the form "#!/opt/python2.3/bin/python", the kernel only passes "python" From 02fb660466c27a8d38a76242579ba8f7f541dd6f Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 21 Jul 2020 13:19:34 +0200 Subject: [PATCH 09/45] First stab at deploying to an earlier version of macOS than the build machine --- Modules/posixmodule.c | 445 +++++++++++++++++++++++++++++++--------- Modules/timemodule.c | 139 ++++++++++++- Python/bootstrap_hash.c | 11 + Python/pytime.c | 25 ++- 4 files changed, 521 insertions(+), 99 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index efd99544f5a997..896f77394441e5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7,18 +7,6 @@ of the compiler used. Different compilers define their own feature test macro, e.g. '_MSC_VER'. */ -#ifdef __APPLE__ - /* - * Step 1 of support for weak-linking a number of symbols existing on - * OSX 10.4 and later, see the comment in the #ifdef __APPLE__ block - * at the end of this file for more information. - */ -# pragma weak lchown -# pragma weak statvfs -# pragma weak fstatvfs - -#endif /* __APPLE__ */ - #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -2370,6 +2358,10 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, STRUCT_STAT st; int result; +#if defined(__APPLE__) && defined(HAVE_FSTATAT) + int fstatat_unavailable = 0; +#endif + #if !defined(MS_WINDOWS) && !defined(HAVE_FSTATAT) && !defined(HAVE_LSTAT) if (follow_symlinks_specified(function_name, follow_symlinks)) return NULL; @@ -2397,14 +2389,29 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, #endif /* HAVE_LSTAT */ #ifdef HAVE_FSTATAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = fstatat(dir_fd, path->narrow, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); + +#ifdef __APPLE__ + } else { + fstatat_unavailable = 1; + } +#endif else #endif /* HAVE_FSTATAT */ result = STAT(path->narrow, &st); #endif /* MS_WINDOWS */ Py_END_ALLOW_THREADS +#if defined(__APPLE__) && defined(HAVE_FSTATAT) + if (fstatat_unavailable) { + argument_unavailable_error("stat", "dir_fd"); + return NULL; + } +#endif if (result != 0) { return path_error(path); } @@ -2822,7 +2829,18 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int result; #endif -#ifndef HAVE_FACCESSAT +#if defined(__APPLE__) && defined(HAVE_FACCESSAT) + int faccessat_unavailable = 0; +#endif + +#if !defined(HAVE_FACCESSAT) || defined(__APPLE__) + +#if defined(HAVE_FACCESSAT) && defined(__APPLE) + if (__builtin_available(macOS 10.10, *)) { + /* ^^^^ cannot use '!' here */ + } else { +#endif + if (follow_symlinks_specified("access", follow_symlinks)) return -1; @@ -2830,6 +2848,11 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, argument_unavailable_error("access", "effective_ids"); return -1; } + +#if defined(HAVE_FACCESSAT) && defined(__APPLE) + } +#endif + #endif #ifdef MS_WINDOWS @@ -2856,17 +2879,31 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, if ((dir_fd != DEFAULT_DIR_FD) || effective_ids || !follow_symlinks) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif int flags = 0; if (!follow_symlinks) flags |= AT_SYMLINK_NOFOLLOW; if (effective_ids) flags |= AT_EACCESS; result = faccessat(dir_fd, path->narrow, mode, flags); +#ifdef __APPLE__ + } else { + faccessat_unavailable = 1; + } +#endif } else #endif result = access(path->narrow, mode); Py_END_ALLOW_THREADS +#if defined(__APPLE__) && defined(HAVE_FACCESSAT) + if (faccessat_unavailable) { + argument_unavailable_error("access", "dir_fd"); + return 0; + } +#endif return_value = !result; #endif @@ -3064,6 +3101,11 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; + +#ifdef __APPLE__ + int fchmodat_unsupported = 0; +#endif + #endif #if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) @@ -3107,6 +3149,9 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #endif #ifdef HAVE_FCHMODAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif /* * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! * The documentation specifically shows how to use it, @@ -3127,6 +3172,15 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, result && ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && !follow_symlinks; +#ifdef __APPLE__ + } else { + fchmodat_unsupported = 1; + + /* These silence a compiler warning, values aren't used */ + errno = ENOSYS; + result = -1; + } +#endif } else #endif @@ -3135,6 +3189,12 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (result) { #ifdef HAVE_FCHMODAT +#ifdef __APPLE__ + if (fchmodat_unsupported) { + argument_unavailable_error("chmod", "dir_fd"); + return NULL; + } else +#endif if (fchmodat_nofollow_unsupported) { if (dir_fd != DEFAULT_DIR_FD) dir_fd_and_follow_symlinks_invalid("chmod", @@ -3435,6 +3495,10 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, { int result; +#if defined(HAVE_FCHOWNAT) && defined(__APPLE__) + int fchownat_unsupported = 0; +#endif + #if !(defined(HAVE_LCHOWN) || defined(HAVE_FCHOWNAT)) if (follow_symlinks_specified("chown", follow_symlinks)) return NULL; @@ -3443,19 +3507,6 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, fd_and_follow_symlinks_invalid("chown", path->fd, follow_symlinks)) return NULL; -#ifdef __APPLE__ - /* - * This is for Mac OS X 10.3, which doesn't have lchown. - * (But we still have an lchown symbol because of weak-linking.) - * It doesn't have fchownat either. So there's no possibility - * of a graceful failover. - */ - if ((!follow_symlinks) && (lchown == NULL)) { - follow_symlinks_specified("chown", follow_symlinks); - return NULL; - } -#endif - if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { return NULL; @@ -3473,14 +3524,29 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, else #endif #ifdef HAVE_FCHOWNAT - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = fchownat(dir_fd, path->narrow, uid, gid, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - else +#ifdef __APPLE__ + } else { + fchownat_unsupported = 1; + } +#endif + } else #endif result = chown(path->narrow, uid, gid); Py_END_ALLOW_THREADS +#ifdef __APPLE__ + if (fchownat_unsupported) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -3726,12 +3792,33 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #else int result; #endif +#if defined(__APPLE__) && defined(HAVE_LINKAT) + int linkat_unavailable = 0; +#endif + +#if !defined(HAVE_LINKAT) || defined(__APPLE__) -#ifndef HAVE_LINKAT +#if defined(HAVE_LINKAT) && defined(__APPLE__) + if (__builtin_available(macOS 10.0, *)) { + /* pass: ^^^^ '!' cannot be used here */ + } else { +#endif if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD)) { argument_unavailable_error("link", "src_dir_fd and dst_dir_fd"); return NULL; } + + /* XXX: !follow_symlinks requires linkat: Issue41355 */ + if (!follow_symlinks) { + argument_unavailable_error("link", "not follow_symlinks"); + return NULL; + } + + +#if defined(HAVE_LINKAT) && defined(__APPLE__) + } +#endif + #endif #ifndef MS_WINDOWS @@ -3761,14 +3848,31 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif + result = linkat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow, follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + +#ifdef __APPLE__ + } else { + linkat_unavailable = 1; + } +#endif else #endif /* HAVE_LINKAT */ result = link(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#if defined(__APPLE__) && defined(HAVE_LINKAT) + if (linkat_unavailable) { + argument_unavailable_error("link", "src_dir_fd, dst_dir_fd, not follow_symlinks"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif /* MS_WINDOWS */ @@ -3891,6 +3995,9 @@ _posix_listdir(path_t *path, PyObject *list) errno = 0; #ifdef HAVE_FDOPENDIR if (path->fd != -1) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(path->fd); if (fd == -1) @@ -3901,6 +4008,12 @@ _posix_listdir(path_t *path, PyObject *list) Py_BEGIN_ALLOW_THREADS dirp = fdopendir(fd); Py_END_ALLOW_THREADS +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif } else #endif @@ -4230,9 +4343,19 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = mkdirat(dir_fd, path->narrow, mode); - else + +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif #if defined(__WATCOMC__) && !defined(__QNX__) result = mkdir(path->narrow); @@ -4388,9 +4511,20 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is Py_BEGIN_ALLOW_THREADS #ifdef HAVE_RENAMEAT - if (dir_fd_specified) + if (dir_fd_specified) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif + result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); - else + +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif result = rename(src->narrow, dst->narrow); Py_END_ALLOW_THREADS @@ -4482,9 +4616,18 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) result = !RemoveDirectoryW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = unlinkat(dir_fd, path->narrow, AT_REMOVEDIR); - else +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif result = rmdir(path->narrow); #endif @@ -4646,9 +4789,19 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) result = !Py_DeleteFileW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif + result = unlinkat(dir_fd, path->narrow, 0); - else +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif /* HAVE_UNLINKAT */ result = unlink(path->narrow); #endif @@ -4822,6 +4975,11 @@ typedef struct { #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) +#ifdef __APPLE__ +static int +utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) __attribute__((availability(macos, introduced=10.13))); +#endif + static int utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { @@ -4852,11 +5010,25 @@ static int utime_fd(utime_t *ut, int fd) { #ifdef HAVE_FUTIMENS + +#ifdef __APPLE__ + if (__builtin_available(macOS 10.13, *)) { +#endif + UTIME_TO_TIMESPEC; return futimens(fd, time); -#else + +#ifdef __APPLE__ + } else +#endif + +#endif + +#if !defined(HAVE_FUTIMENS) || defined(__APPLE__) + { UTIME_TO_TIMEVAL; return futimes(fd, time); + } #endif } @@ -4875,11 +5047,21 @@ static int utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT +#ifdef __APPLE__ + if (__builtin_available(macOS 10.13, *)) { +#endif UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); -#else +#ifdef __APPLE__ + } else +#endif +#endif + +#if !defined(HAVE_UTIMENSAT) || defined(__APPLE__) + { UTIME_TO_TIMEVAL; return lutimes(path, time); + } #endif } @@ -4890,7 +5072,7 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) static int utime_default(utime_t *ut, const char *path) { -#ifdef HAVE_UTIMENSAT +#if defined(HAVE_UTIMENSAT) && !defined(__APPLE__) /* XXX */ UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, 0); #elif defined(HAVE_UTIMES) @@ -4984,6 +5166,9 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #else int result; #endif +#if defined(__APPLE__) && (defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT)) + int utime_dir_fd_unsupported = 0; +#endif utime_t utime; @@ -5099,9 +5284,19 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #endif #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.13, *)) { +#endif result = utime_dir_fd(&utime, dir_fd, path->narrow, follow_symlinks); - else + +#ifdef __APPLE__ + } else { + utime_dir_fd_unsupported = 1; + } +#endif + + } else #endif #if defined(HAVE_FUTIMES) || defined(HAVE_FUTIMENS) @@ -5114,6 +5309,13 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, Py_END_ALLOW_THREADS +#if defined(__APPLE__) && (defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT)) + if (utime_dir_fd_unsupported) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) { /* see previous comment about not putting filename in error here */ posix_error(); @@ -8133,9 +8335,18 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) Py_BEGIN_ALLOW_THREADS #ifdef HAVE_READLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); - else +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif length = readlink(path->narrow, buffer, MAXPATHLEN); Py_END_ALLOW_THREADS @@ -8398,9 +8609,18 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, Py_BEGIN_ALLOW_THREADS #if HAVE_SYMLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = symlinkat(src->narrow, dir_fd, dst->narrow); - else +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + } else #endif result = symlink(src->narrow, dst->narrow); Py_END_ALLOW_THREADS @@ -8699,9 +8919,19 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) fd = _wopen(path->wide, flags, mode); #else #ifdef HAVE_OPENAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif fd = openat(dir_fd, path->narrow, flags, mode); - else + +#ifdef __APPLE__ + } else { + argument_unavailable_error(NULL, "dir_fd"); + return -1; + } +#endif + } else #endif /* HAVE_OPENAT */ fd = open(path->narrow, flags, mode); #endif /* !MS_WINDOWS */ @@ -9291,12 +9521,25 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else do { +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH n = preadv(fd, iov, cnt, offset); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -9886,6 +10129,15 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else + +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH @@ -9893,6 +10145,11 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -10782,13 +11039,6 @@ os_statvfs_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS #ifdef HAVE_FSTATVFS if (path->fd != -1) { -#ifdef __APPLE__ - /* handle weak-linking on Mac OS X 10.3 */ - if (fstatvfs == NULL) { - fd_specified("statvfs", path->fd); - return NULL; - } -#endif result = fstatvfs(path->fd, &st); } else @@ -12870,12 +13120,23 @@ _Py_COMP_DIAG_POP const char *path = PyBytes_AS_STRING(ub); if (self->dir_fd != DEFAULT_DIR_FD) { #ifdef HAVE_FSTATAT +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif result = fstatat(self->dir_fd, path, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#else +#ifdef __APPLE__ + } else +#endif + +#endif + +#if !defined(HAVE_FSTATAT) || defined(__APPLE__) + { Py_DECREF(ub); PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat"); return NULL; + } #endif /* HAVE_FSTATAT */ } else @@ -13675,6 +13936,9 @@ os_scandir_impl(PyObject *module, path_t *path) errno = 0; #ifdef HAVE_FDOPENDIR if (iterator->path.fd != -1) { +#ifdef __APPLE__ + if (__builtin_available(macOS 10.10, *)) { +#endif /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(iterator->path.fd); if (fd == -1) @@ -13683,6 +13947,12 @@ os_scandir_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS iterator->dirp = fdopendir(fd); Py_END_ALLOW_THREADS +#ifdef __APPLE__ + } else { + argument_unavailable_error("opendir", "dir_fd"); + return NULL; + } +#endif } else #endif @@ -15002,44 +15272,6 @@ posixmodule_exec(PyObject *m) PyModule_AddObject(m, "uname_result", (PyObject *)UnameResultType); state->UnameResultType = (PyObject *)UnameResultType; -#ifdef __APPLE__ - /* - * Step 2 of weak-linking support on Mac OS X. - * - * The code below removes functions that are not available on the - * currently active platform. - * - * This block allow one to use a python binary that was build on - * OSX 10.4 on OSX 10.3, without losing access to new APIs on - * OSX 10.4. - */ -#ifdef HAVE_FSTATVFS - if (fstatvfs == NULL) { - if (PyObject_DelAttrString(m, "fstatvfs") == -1) { - return -1; - } - } -#endif /* HAVE_FSTATVFS */ - -#ifdef HAVE_STATVFS - if (statvfs == NULL) { - if (PyObject_DelAttrString(m, "statvfs") == -1) { - return -1; - } - } -#endif /* HAVE_STATVFS */ - -# ifdef HAVE_LCHOWN - if (lchown == NULL) { - if (PyObject_DelAttrString(m, "lchown") == -1) { - return -1; - } - } -#endif /* HAVE_LCHOWN */ - - -#endif /* __APPLE__ */ - if ((state->billion = PyLong_FromLong(1000000000)) == NULL) return -1; #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -15103,7 +15335,30 @@ static struct PyModuleDef posixmodule = { PyMODINIT_FUNC INITFUNC(void) { - return PyModuleDef_Init(&posixmodule); + PyObject* module = PyModuleDef_Init(&posixmodule); +#if defined(__APPLE__) && defined(HAVE_PWRITEV) + if (module) { + if (__builtin_available(macOS 11.0, *)) { + /* pass: ^^^^ cannot use '!' here */ + } else { + /* pwritev and preadv were introduced in macOS 11 */ + PyObject* dct = PyModule_GetDict(module); + + if (dct == NULL) { + return NULL; + } + + if (PyDict_DelItemString(dct, "pwritev") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "preadv") == -1) { + PyErr_Clear(); + } + } + } +#endif + + return module; } #ifdef __cplusplus diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 8a4d149befb52a..a97a6a18ec5d1d 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -149,6 +149,16 @@ perf_counter(_Py_clock_info_t *info) } #ifdef HAVE_CLOCK_GETTIME + +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_clock_gettime(PyObject *self, PyObject *args) { @@ -297,6 +307,11 @@ PyDoc_STRVAR(clock_getres_doc, "clock_getres(clk_id) -> floating point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif /* HAVE_CLOCK_GETRES */ #ifdef HAVE_PTHREAD_GETCPUCLOCKID @@ -1162,6 +1177,11 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #if defined(HAVE_CLOCK_GETTIME) \ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) struct timespec ts; + +#ifdef __APPLE__ + if (__builtin_available(macos 10.12, *)) { +#endif + #ifdef CLOCK_PROF const clockid_t clk_id = CLOCK_PROF; const char *function = "clock_gettime(CLOCK_PROF)"; @@ -1188,6 +1208,9 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) } return 0; } +#ifdef __APPLE__ + } +#endif #endif /* getrusage(RUSAGE_SELF) */ @@ -1373,6 +1396,12 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) #define HAVE_THREAD_TIME + +#ifdef __APPLE__ +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))); +#endif + static int _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { @@ -1404,6 +1433,15 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif #ifdef HAVE_THREAD_TIME +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_thread_time(PyObject *self, PyObject *unused) { @@ -1434,6 +1472,11 @@ PyDoc_STRVAR(thread_time_ns_doc, \n\ Thread time for profiling as nanoseconds:\n\ sum of the kernel and user-space CPU time."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif @@ -1483,9 +1526,19 @@ time_get_clock_info(PyObject *self, PyObject *args) } #ifdef HAVE_THREAD_TIME else if (strcmp(name, "thread_time") == 0) { + +#ifdef __APPLE__ + if (__builtin_available(macos 10.12, *)) { +#endif if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { return NULL; } +#ifdef __APPLE__ + } else { + PyErr_SetString(PyExc_ValueError, "unknown clock"); + return NULL; + } +#endif } #endif else { @@ -1774,34 +1827,70 @@ time_exec(PyObject *module) #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) #ifdef CLOCK_REALTIME +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { return -1; } +#ifdef __APPLE__ + } +#endif #endif + #ifdef CLOCK_MONOTONIC + +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { return -1; } + +#ifdef __APPLE__ + } +#endif + #endif #ifdef CLOCK_MONOTONIC_RAW +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { return -1; } +#ifdef __APPLE__ + } +#endif #endif + #ifdef CLOCK_HIGHRES if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { return -1; } #endif #ifdef CLOCK_PROCESS_CPUTIME_ID +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { return -1; } +#ifdef __APPLE__ + } #endif +#endif + #ifdef CLOCK_THREAD_CPUTIME_ID +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { return -1; } +#ifdef __APPLE__ + } +#endif #endif #ifdef CLOCK_PROF if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { @@ -1824,9 +1913,17 @@ time_exec(PyObject *module) } #endif #ifdef CLOCK_UPTIME_RAW + +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { return -1; } + +#ifdef __APPLE__ + } +#endif #endif #endif /* defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) */ @@ -1877,7 +1974,47 @@ static struct PyModuleDef timemodule = { PyMODINIT_FUNC PyInit_time(void) { - return PyModuleDef_Init(&timemodule); + PyObject* module = PyModuleDef_Init(&timemodule); + +#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) + if (__builtin_available(macOS 10.12, *)) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "clock_gettime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_getres") == -1) { + PyErr_Clear(); + } + } +#endif +#if defined(__APPLE__) && defined(HAVE_THREAD_TIME) + if (__builtin_available(macOS 10.12, *)) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "thread_time") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { + PyErr_Clear(); + } + } +#endif + + return module; } /* Implement pysleep() for various platforms. diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 47369305ee88e9..d10019e5b59b5d 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -208,6 +208,11 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) error. getentropy() is retried if it failed with EINTR: interrupted by a signal. */ +#ifdef __APPLE__ +static int +py_getentropy(char *buffer, Py_ssize_t size, int raise) __attribute__((availability(macos,introduced=10.12))); +#endif + static int py_getentropy(char *buffer, Py_ssize_t size, int raise) { @@ -498,6 +503,9 @@ pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise) #else #if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) +#ifdef __APPLE__ + if (__builtin_available(macOS 10.12, *)) { +#endif #ifdef PY_GETRANDOM res = py_getrandom(buffer, size, blocking, raise); #else @@ -511,6 +519,9 @@ pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise) } /* getrandom() or getentropy() function is not available: failed with ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ +#ifdef __APPLE__ + } /* end of availability block */ +#endif #endif return dev_urandom(buffer, size, raise); diff --git a/Python/pytime.c b/Python/pytime.c index b121b432f428d7..a1b49ff26ab294 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -683,15 +683,22 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) #else /* MS_WINDOWS */ int err; -#ifdef HAVE_CLOCK_GETTIME +#if defined(HAVE_CLOCK_GETTIME) struct timespec ts; -#else +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(__APPLE__) struct timeval tv; #endif assert(info == NULL || raise); #ifdef HAVE_CLOCK_GETTIME + +#if defined(__APPLE__) + if (__builtin_available(macOS 10.12, *)) { +#endif + err = clock_gettime(CLOCK_REALTIME, &ts); if (err) { if (raise) { @@ -715,7 +722,14 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->resolution = 1e-9; } } -#else /* HAVE_CLOCK_GETTIME */ + +#ifdef __APPLE__ + } else { +#endif + +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(__APPLE__) /* test gettimeofday() */ err = gettimeofday(&tv, (struct timezone *)NULL); @@ -735,6 +749,11 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->monotonic = 0; info->adjustable = 1; } + +#ifdef __APPLE__ + } /* end of availibity block */ +#endif + #endif /* !HAVE_CLOCK_GETTIME */ #endif /* !MS_WINDOWS */ return 0; From ba2f5a324fba1f16c0c4f4e98fd906e5a470bb69 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 14:52:51 +0200 Subject: [PATCH 10/45] Update build-installer.py for universal2 builds This also adds an option to stop building after compiling the 3th-party dependencies, as well as a script for archiving those dependencies. This makes it easier to work on the build. There are three changes to build-installer.py related to universal2 support: 1. Add 'universal2' information to build-installer.py; 2. Building OpenSSL for arm64 requires a patch at this time; 3. For some reason I had to patch the Tcl build to avoid a build error. --- Mac/BuildScript/archive-deps.py | 39 ++++++++++++++++++++ Mac/BuildScript/build-installer.py | 47 +++++++++++++++++++++++-- Mac/BuildScript/openssl-mac-arm64.patch | 41 +++++++++++++++++++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100755 Mac/BuildScript/archive-deps.py create mode 100644 Mac/BuildScript/openssl-mac-arm64.patch diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py new file mode 100755 index 00000000000000..06d73a0f07617c --- /dev/null +++ b/Mac/BuildScript/archive-deps.py @@ -0,0 +1,39 @@ +#!/usr/bin/python + +import shutil +import getopt +import os +import sys + +WORKDIR = "/tmp/_py" + +def main(): + workdir = WORKDIR + + try: + options, args = getopt.getopt(sys.argv[1:], "?hbo", + [ "build-dir=", "output=", "help" ]) + except getopt.GetoptError: + print(sys.exc_info()[1]) + sys.exit(1) + + if args: + print("Additional arguments") + sys.exit(1) + + for k, v in options: + if k in ("-h", "-?", "--help"): + print(USAGE) + sys.exit(0) + + elif k in ("b", "--build-dir"): + workdir=v + + else: + raise SystemExit("Unknown option") + + root = os.path.join(workdir, "libraries", "Library", "Frameworks") + shutil.make_archive("third-party", "zip", root_dir=root) + +if __name__ == "__main__": + main() diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index a58b922ce30b83..56fc20390c3f97 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ def getFullVersion(): '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -148,6 +150,10 @@ def getFullVersion(): # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level DEPTARGET = '10.5' +# If true only builds the 3th-party dependencies +# in $WORKDIR +DEPS_ONLY=False + def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) @@ -190,6 +196,27 @@ def getTargetCompilers(): def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +242,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +261,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -596,7 +627,7 @@ def checkEnvironment(): ev, os.environ[ev])) del os.environ[ev] - base_path = '/bin:/sbin:/usr/bin:/usr/sbin' + base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' if 'SDK_TOOLS_BIN' in os.environ: base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; @@ -618,6 +649,7 @@ def parseOptions(args=None): global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY + global DEPS_ONLY if args is None: args = sys.argv[1:] @@ -625,7 +657,7 @@ def parseOptions(args=None): try: options, args = getopt.getopt(args, '?hb', [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', - 'dep-target=', 'universal-archs=', 'help' ]) + 'dep-target=', 'universal-archs=', 'deps-only', 'help' ]) except getopt.GetoptError: print(sys.exc_info()[1]) sys.exit(1) @@ -646,6 +678,9 @@ def parseOptions(args=None): elif k in ('--third-party',): DEPSRC=v + elif k in ('--deps-only',): + DEPS_ONLY=True + elif k in ('--sdk-path',): print(" WARNING: --sdk-path is no longer supported") @@ -691,6 +726,8 @@ def parseOptions(args=None): print(" -- Building a Python %s framework at patch level %s" % (getVersion(), getFullVersion())) print("") + if DEPS_ONLY: + print("Stopping after building third-party libraries") def extractArchive(builddir, archiveName): """ @@ -801,6 +838,7 @@ def build_openssl_arch(archbase, arch): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc" ], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } @@ -1656,6 +1694,9 @@ def main(): # Then build third-party libraries such as sleepycat DB4. buildLibraries() + if DEPS_ONLY: + sys.exit(1) + # Now build python itself buildPython() diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch new file mode 100644 index 00000000000000..11267fb118744a --- /dev/null +++ b/Mac/BuildScript/openssl-mac-arm64.patch @@ -0,0 +1,41 @@ +diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf +--- openssl-1.1.1g-orig/Configurations/10-main.conf 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/Configurations/10-main.conf 2020-07-26 12:21:32.000000000 +0200 +@@ -1557,6 +1557,14 @@ + bn_ops => "SIXTY_FOUR_BIT_LONG", + perlasm_scheme => "macosx", + }, ++ "darwin64-arm64-cc" => { ++ inherit_from => [ "darwin-common", asm("aarch64_asm") ], ++ CFLAGS => add("-Wall"), ++ cflags => add("-arch arm64"), ++ lib_cppflags => add("-DL_ENDIAN"), ++ bn_ops => "SIXTY_FOUR_BIT_LONG", ++ perlasm_scheme => "ios64", ++ }, + + ##### GNU Hurd + "hurd-x86" => { +diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config +--- openssl-1.1.1g-orig/config 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/config 2020-07-26 12:21:59.000000000 +0200 +@@ -255,6 +255,9 @@ + ;; + x86_64) + echo "x86_64-apple-darwin${VERSION}" ++ ;; ++ arm64) ++ echo "arm64-apple-darwin${VERSION}" + ;; + *) + echo "i686-apple-darwin${VERSION}" +@@ -497,6 +500,9 @@ + else + OUT="darwin64-x86_64-cc" + fi ;; ++ x86_64-apple-darwin*) ++ OUT="darwin64-arm64-cc" ++ ;; + armv6+7-*-iphoneos) + __CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7" + __CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7" From 8e3b4548f3843518996a9b754f213740479f290d Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:06:43 +0200 Subject: [PATCH 11/45] Merge ctypes changes from Apple (PR 21241) Needed because my previous workaround doesn't work anymore. This uses a private API, that should be made public later... --- Lib/ctypes/macholib/dyld.py | 24 ++++++++-------- Lib/ctypes/test/test_macholib.py | 15 ++++++---- Modules/_ctypes/callproc.c | 48 ++++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9ab9e79ab816c4..22825d0d15b290 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -123,19 +128,14 @@ def dyld_find(name, executable_path=None, env=None): dyld_default_search(name, env), ), env): - # on macOS 11 system libraries are in a shared library - # cache, not in the regular place in the filesystem. - # There still are symlinks (libz.dylib -> libz.1.dylib), - # but the target of the symlink no longer is there. - # - # There shouldn't be 3th-party libraries in these - # system locations - if path.startswith("/System/") and os.path.islink(path): - return path - elif path.startswith("/usr/lib/") and os.path.islink(path): - return path - elif os.path.isfile(path): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951acfab..a1bac26a7df058 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ac407c93c240e4..824b00af0e60c2 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,6 +66,18 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +/* + * The API to query if a shared library is in the shared cache is + * private for now, this should change in beta 4. + * + * TODO: + * - Switch to that API + * - Add feature macro and runtime guards (as with the ffi.*loc API's) + */ +extern bool _dyld_shared_cache_contains_path(const char* path) __attribute__((weak_import)); +#endif + #ifdef MS_WIN32 #include #endif @@ -1446,6 +1458,37 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else +#ifdef __APPLE__ + static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) + { + PyObject *name, *name2; + char *name_str; + + if (_dyld_shared_cache_contains_path == NULL) { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + if (PyBytes_Check(name2)) + name_str = PyBytes_AS_STRING(name2); + else + name_str = PyByteArray_AS_STRING(name2); + + if(_dyld_shared_cache_contains_path(name_str)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + } + #endif + static PyObject *py_dl_open(PyObject *self, PyObject *args) { PyObject *name, *name2; @@ -1935,6 +1978,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1956,6 +2001,9 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#endif +#ifdef __APPLE__ + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, diff --git a/setup.py b/setup.py index 65d479fcb184c6..3bbf17a1c2283e 100644 --- a/setup.py +++ b/setup.py @@ -2211,6 +2211,7 @@ def detect_ctypes(self): ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") if grep_headers_for('ffi_closure_alloc', ffi_headers): ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 87c942becc00ac0c4bcb06f049e69aa2df8622e0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:36:23 +0200 Subject: [PATCH 12/45] The archive-deps.py script isn't really useful --- Mac/BuildScript/archive-deps.py | 39 --------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 Mac/BuildScript/archive-deps.py diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py deleted file mode 100755 index 06d73a0f07617c..00000000000000 --- a/Mac/BuildScript/archive-deps.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python - -import shutil -import getopt -import os -import sys - -WORKDIR = "/tmp/_py" - -def main(): - workdir = WORKDIR - - try: - options, args = getopt.getopt(sys.argv[1:], "?hbo", - [ "build-dir=", "output=", "help" ]) - except getopt.GetoptError: - print(sys.exc_info()[1]) - sys.exit(1) - - if args: - print("Additional arguments") - sys.exit(1) - - for k, v in options: - if k in ("-h", "-?", "--help"): - print(USAGE) - sys.exit(0) - - elif k in ("b", "--build-dir"): - workdir=v - - else: - raise SystemExit("Unknown option") - - root = os.path.join(workdir, "libraries", "Library", "Frameworks") - shutil.make_archive("third-party", "zip", root_dir=root) - -if __name__ == "__main__": - main() From eee54372264dbabc1b5d91c958c51b6b30ad30a3 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 20 Oct 2020 20:59:10 +0200 Subject: [PATCH 13/45] Cleanup the code to check at runtime for a couple of APIs. --- Modules/posixmodule.c | 545 ++++++++++++++++++++++------------------ Modules/timemodule.c | 178 ++++++------- Python/bootstrap_hash.c | 44 ++-- Python/pytime.c | 2 +- 4 files changed, 400 insertions(+), 369 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 896f77394441e5..b329b21cfd41b9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -39,6 +39,61 @@ #include /* needed for ctermid() */ +/* + * A number of APIs are available on macOS from a certain macOS version. + * To support building with a new SDK while deploying to older versions + * the availability test is split into two: + * - HAVE_: The configure check for compile time availability + * - HAVE__RUNTIME: Runtime check for availability + * + * The latter is always true when not on macOS, or when using a compiler + * that does not support __has_builtin (older versions of Xcode). + * + * Due to compiler restrictions there is one valid use of HAVE__RUNTIME: + * if (HAVE__RUNTIME) { ... } + * + * In mixing the test with other tests or using negations will result in compile + * errors. + */ +#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_FSTATAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FACCESSAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHMODAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHOWNAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_LINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FDOPENDIR_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_MKDIRAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_RENAMEAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_UNLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_OPENAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_READLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_SYMLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FUTIMENS_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_UTIMENSAT_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + +#ifdef HAVE_FUTIMESAT +# error "HAVE_FUTIMESAT unexpectedly defined" +#endif + +#else +# define HAVE_FSTATAT_RUNTIME 1 +# define HAVE_FACCESSAT_RUNTIME 1 +# define HAVE_FCHMODAT_RUNTIME 1 +# define HAVE_FCHOWNAT_RUNTIME 1 +# define HAVE_LINKAT_RUNTIME 1 +# define HAVE_FDOPENDIR_RUNTIME 1 +# define HAVE_MKDIRAT_RUNTIME 1 +# define HAVE_RENAMEAT_RUNTIME 1 +# define HAVE_UNLINKAT_RUNTIME 1 +# define HAVE_OPENAT_RUNTIME 1 +# define HAVE_READLINKAT_RUNTIME 1 +# define HAVE_SYMLINKAT_RUNTIME 1 +# define HAVE_FUTIMENS_RUNTIME 1 +# define HAVE_UTIMENSAT_RUNTIME 1 +# define HAVE_PWRITEV_RUNTIME 1 +#endif + #ifdef __cplusplus extern "C" { #endif @@ -2358,7 +2413,7 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, STRUCT_STAT st; int result; -#if defined(__APPLE__) && defined(HAVE_FSTATAT) +#if defined(HAVE_FSTATAT) int fstatat_unavailable = 0; #endif @@ -2388,30 +2443,25 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, else #endif /* HAVE_LSTAT */ #ifdef HAVE_FSTATAT - if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - result = fstatat(dir_fd, path->narrow, &st, + if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { + if (HAVE_FSTATAT_RUNTIME) { + result = fstatat(dir_fd, path->narrow, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#ifdef __APPLE__ - } else { - fstatat_unavailable = 1; - } -#endif - else + } else { + fstatat_unavailable = 1; + } + } else #endif /* HAVE_FSTATAT */ result = STAT(path->narrow, &st); #endif /* MS_WINDOWS */ Py_END_ALLOW_THREADS -#if defined(__APPLE__) && defined(HAVE_FSTATAT) if (fstatat_unavailable) { argument_unavailable_error("stat", "dir_fd"); return NULL; } -#endif + if (result != 0) { return path_error(path); } @@ -2829,18 +2879,11 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int result; #endif -#if defined(__APPLE__) && defined(HAVE_FACCESSAT) +#ifdef HAVE_FACCESSAT int faccessat_unavailable = 0; #endif -#if !defined(HAVE_FACCESSAT) || defined(__APPLE__) - -#if defined(HAVE_FACCESSAT) && defined(__APPLE) - if (__builtin_available(macOS 10.10, *)) { - /* ^^^^ cannot use '!' here */ - } else { -#endif - +#ifndef HAVE_FACCESSAT if (follow_symlinks_specified("access", follow_symlinks)) return -1; @@ -2848,11 +2891,6 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, argument_unavailable_error("access", "effective_ids"); return -1; } - -#if defined(HAVE_FACCESSAT) && defined(__APPLE) - } -#endif - #endif #ifdef MS_WINDOWS @@ -2879,29 +2917,38 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, if ((dir_fd != DEFAULT_DIR_FD) || effective_ids || !follow_symlinks) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - int flags = 0; - if (!follow_symlinks) - flags |= AT_SYMLINK_NOFOLLOW; - if (effective_ids) - flags |= AT_EACCESS; - result = faccessat(dir_fd, path->narrow, mode, flags); -#ifdef __APPLE__ - } else { - faccessat_unavailable = 1; - } -#endif + + if (HAVE_FACCESSAT_RUNTIME) { + int flags = 0; + if (!follow_symlinks) + flags |= AT_SYMLINK_NOFOLLOW; + if (effective_ids) + flags |= AT_EACCESS; + result = faccessat(dir_fd, path->narrow, mode, flags); + } else { + faccessat_unavailable = 1; + } } else #endif result = access(path->narrow, mode); Py_END_ALLOW_THREADS -#if defined(__APPLE__) && defined(HAVE_FACCESSAT) + +#if defined(HAVE_FACCESSAT) if (faccessat_unavailable) { - argument_unavailable_error("access", "dir_fd"); - return 0; + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("access", "dir_fd"); + return -1; + } + if (follow_symlinks_specified("access", follow_symlinks)) + return -1; + + if (effective_ids) { + argument_unavailable_error("access", "effective_ids"); + return -1; + } + /* should be unreachable */ + return -1; } #endif return_value = !result; @@ -3101,13 +3148,9 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; - -#ifdef __APPLE__ int fchmodat_unsupported = 0; #endif -#endif - #if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) if (follow_symlinks_specified("chmod", follow_symlinks)) return NULL; @@ -3149,37 +3192,33 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #endif #ifdef HAVE_FCHMODAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - /* - * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! - * The documentation specifically shows how to use it, - * and then says it isn't implemented yet. - * (true on linux with glibc 2.15, and openindiana 3.x) - * - * Once it is supported, os.chmod will automatically - * support dir_fd and follow_symlinks=False. (Hopefully.) - * Until then, we need to be careful what exception we raise. - */ - result = fchmodat(dir_fd, path->narrow, mode, - follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - /* - * But wait! We can't throw the exception without allowing threads, - * and we can't do that in this nested scope. (Macro trickery, sigh.) - */ - fchmodat_nofollow_unsupported = - result && - ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && - !follow_symlinks; -#ifdef __APPLE__ - } else { - fchmodat_unsupported = 1; + if (HAVE_FCHMODAT_RUNTIME) { + /* + * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! + * The documentation specifically shows how to use it, + * and then says it isn't implemented yet. + * (true on linux with glibc 2.15, and openindiana 3.x) + * + * Once it is supported, os.chmod will automatically + * support dir_fd and follow_symlinks=False. (Hopefully.) + * Until then, we need to be careful what exception we raise. + */ + result = fchmodat(dir_fd, path->narrow, mode, + follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); + /* + * But wait! We can't throw the exception without allowing threads, + * and we can't do that in this nested scope. (Macro trickery, sigh.) + */ + fchmodat_nofollow_unsupported = + result && + ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && + !follow_symlinks; + } else { + fchmodat_unsupported = 1; + fchmodat_nofollow_unsupported = 1; - /* These silence a compiler warning, values aren't used */ - errno = ENOSYS; - result = -1; - } + result = -1; + } #endif } else @@ -3189,12 +3228,13 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (result) { #ifdef HAVE_FCHMODAT -#ifdef __APPLE__ if (fchmodat_unsupported) { - argument_unavailable_error("chmod", "dir_fd"); - return NULL; - } else -#endif + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("chmod", "dir_fd"); + return NULL; + } + } + if (fchmodat_nofollow_unsupported) { if (dir_fd != DEFAULT_DIR_FD) dir_fd_and_follow_symlinks_invalid("chmod", @@ -3207,7 +3247,6 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #endif return path_error(path); } -#endif Py_RETURN_NONE; } @@ -3495,7 +3534,7 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, { int result; -#if defined(HAVE_FCHOWNAT) && defined(__APPLE__) +#if defined(HAVE_FCHOWNAT) int fchownat_unsupported = 0; #endif @@ -3525,23 +3564,22 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, #endif #ifdef HAVE_FCHOWNAT if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_FCHOWNAT_RUNTIME) { result = fchownat(dir_fd, path->narrow, uid, gid, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#ifdef __APPLE__ } else { fchownat_unsupported = 1; } -#endif } else #endif result = chown(path->narrow, uid, gid); Py_END_ALLOW_THREADS -#ifdef __APPLE__ +#ifdef HAVE_FCHOWNAT if (fchownat_unsupported) { + /* This would be incorrect if the current platform + * doesn't support lchown. + */ argument_unavailable_error(NULL, "dir_fd"); return NULL; } @@ -3792,33 +3830,15 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #else int result; #endif -#if defined(__APPLE__) && defined(HAVE_LINKAT) +#if defined(HAVE_LINKAT) int linkat_unavailable = 0; #endif -#if !defined(HAVE_LINKAT) || defined(__APPLE__) - -#if defined(HAVE_LINKAT) && defined(__APPLE__) - if (__builtin_available(macOS 10.0, *)) { - /* pass: ^^^^ '!' cannot be used here */ - } else { -#endif +#ifndef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD)) { argument_unavailable_error("link", "src_dir_fd and dst_dir_fd"); return NULL; } - - /* XXX: !follow_symlinks requires linkat: Issue41355 */ - if (!follow_symlinks) { - argument_unavailable_error("link", "not follow_symlinks"); - return NULL; - } - - -#if defined(HAVE_LINKAT) && defined(__APPLE__) - } -#endif - #endif #ifndef MS_WINDOWS @@ -3847,28 +3867,38 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #ifdef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD) || - (!follow_symlinks)) -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + (!follow_symlinks)) { - result = linkat(src_dir_fd, src->narrow, - dst_dir_fd, dst->narrow, - follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + if (HAVE_LINKAT_RUNTIME) { + result = linkat(src_dir_fd, src->narrow, + dst_dir_fd, dst->narrow, + follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + #ifdef __APPLE__ - } else { - linkat_unavailable = 1; - } + } else { + if (src_dir_fd == DEFAULT_DIR_FD && dst_dir_fd == DEFAULT_DIR_FD) { + /* See issue 41355: This matches the behaviour of !HAVE_LINKAT */ + result = link(src->narrow, dst->narrow); + } else { + linkat_unavailable = 1; + } + } + } #endif else #endif /* HAVE_LINKAT */ result = link(src->narrow, dst->narrow); Py_END_ALLOW_THREADS -#if defined(__APPLE__) && defined(HAVE_LINKAT) +#ifdef HAVE_LINKAT if (linkat_unavailable) { - argument_unavailable_error("link", "src_dir_fd, dst_dir_fd, not follow_symlinks"); + /* Either or both dir_fd arguments were specified */ + if (src_dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("link", "src_dir_fd"); + } else { + argument_unavailable_error("link", "dst_dir_fd"); + } return NULL; } #endif @@ -3995,9 +4025,7 @@ _posix_listdir(path_t *path, PyObject *list) errno = 0; #ifdef HAVE_FDOPENDIR if (path->fd != -1) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(path->fd); if (fd == -1) @@ -4008,12 +4036,11 @@ _posix_listdir(path_t *path, PyObject *list) Py_BEGIN_ALLOW_THREADS dirp = fdopendir(fd); Py_END_ALLOW_THREADS -#ifdef __APPLE__ } else { - argument_unavailable_error(NULL, "dir_fd"); + /* XXX: This error message is suboptimal... */ + argument_unavailable_error(NULL, "path as file descriptor"); return NULL; } -#endif } else #endif @@ -4327,6 +4354,9 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/ { int result; +#ifdef HAVE_MKDIRAT + int mkdirat_unavailable = 0; +#endif if (PySys_Audit("os.mkdir", "Oii", path->object, mode, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4344,17 +4374,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_MKDIRAT_RUNTIME) { result = mkdirat(dir_fd, path->narrow, mode); -#ifdef __APPLE__ } else { - argument_unavailable_error(NULL, "dir_fd"); - return NULL; + mkdirat_unavailable = 1; } -#endif } else #endif #if defined(__WATCOMC__) && !defined(__QNX__) @@ -4363,6 +4388,14 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) result = mkdir(path->narrow, mode); #endif Py_END_ALLOW_THREADS + +#if HAVE_MKDIRAT + if (mkdirat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) return path_error(path); #endif /* MS_WINDOWS */ @@ -4472,6 +4505,10 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is const char *function_name = is_replace ? "replace" : "rename"; int dir_fd_specified; +#ifdef HAVE_RENAMEAT + int renameat_unavailable = 0; +#endif + #ifdef MS_WINDOWS BOOL result; int flags = is_replace ? MOVEFILE_REPLACE_EXISTING : 0; @@ -4512,22 +4549,23 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is Py_BEGIN_ALLOW_THREADS #ifdef HAVE_RENAMEAT if (dir_fd_specified) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { + if (HAVE_RENAMEAT_RUNTIME) { + result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); + } else { + renameat_unavailable = 1; + } + } else #endif + result = rename(src->narrow, dst->narrow); + Py_END_ALLOW_THREADS - result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); -#ifdef __APPLE__ - } else { - argument_unavailable_error(NULL, "dir_fd"); +#ifdef HAVE_RENAMEAT + if (renameat_unavailable) { + argument_unavailable_error(function_name, "dir_fd"); return NULL; - } -#endif - } else + } #endif - result = rename(src->narrow, dst->narrow); - Py_END_ALLOW_THREADS if (result) return path_error2(src, dst); @@ -4604,6 +4642,9 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.rmdir", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4617,22 +4658,25 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) #else #ifdef HAVE_UNLINKAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_UNLINKAT_RUNTIME) { result = unlinkat(dir_fd, path->narrow, AT_REMOVEDIR); -#ifdef __APPLE__ } else { - argument_unavailable_error(NULL, "dir_fd"); - return NULL; + unlinkat_unavailable = 1; + result = -1; } #endif } else -#endif result = rmdir(path->narrow); #endif Py_END_ALLOW_THREADS +#ifdef HAVE_ULINKAT + if (unlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4776,6 +4820,9 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.remove", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4790,17 +4837,12 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) #else #ifdef HAVE_UNLINKAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_UNLINKAT_RUNTIME) { result = unlinkat(dir_fd, path->narrow, 0); -#ifdef __APPLE__ } else { - argument_unavailable_error(NULL, "dir_fd"); - return NULL; + unlinkat_unavailable = 1; } -#endif } else #endif /* HAVE_UNLINKAT */ result = unlink(path->narrow); @@ -4808,6 +4850,13 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS +#ifdef HAVE_UNLINKAT + if (unlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4975,15 +5024,19 @@ typedef struct { #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) -#ifdef __APPLE__ -static int -utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) __attribute__((availability(macos, introduced=10.13))); -#endif - static int utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { -#ifdef HAVE_UTIMENSAT +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; + UTIME_TO_TIMESPEC; + return utimensat(dir_fd, path, time, flags); + } else { + errno = ENOSYS; + return -1; + } +#elif defined(HAVE_UTIMENSAT) int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; UTIME_TO_TIMESPEC; return utimensat(dir_fd, path, time, flags); @@ -5011,20 +5064,25 @@ utime_fd(utime_t *ut, int fd) { #ifdef HAVE_FUTIMENS -#ifdef __APPLE__ - if (__builtin_available(macOS 10.13, *)) { -#endif + if (HAVE_FUTIMENS_RUNTIME) { UTIME_TO_TIMESPEC; return futimens(fd, time); -#ifdef __APPLE__ } else +#ifndef HAVE_FUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither futimens nor futimes are supported" + " on this system"); + return -1; + } #endif #endif - -#if !defined(HAVE_FUTIMENS) || defined(__APPLE__) +#ifdef HAVE_FUTIMES { UTIME_TO_TIMEVAL; return futimes(fd, time); @@ -5047,17 +5105,23 @@ static int utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT -#ifdef __APPLE__ - if (__builtin_available(macOS 10.13, *)) { -#endif - UTIME_TO_TIMESPEC; - return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); -#ifdef __APPLE__ + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); } else +#ifndef HAVE_LUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither utimensat nor lutimes are supported" + " on this system"); + return -1; + } #endif #endif -#if !defined(HAVE_UTIMENSAT) || defined(__APPLE__) +#ifdef HAVE_LUTIMES { UTIME_TO_TIMEVAL; return lutimes(path, time); @@ -5072,7 +5136,15 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) static int utime_default(utime_t *ut, const char *path) { -#if defined(HAVE_UTIMENSAT) && !defined(__APPLE__) /* XXX */ +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, 0); + } else { + UTIME_TO_TIMEVAL; + return utimes(path, time); + } +#elif defined(HAVE_UTIMENSAT) UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, 0); #elif defined(HAVE_UTIMES) @@ -5166,9 +5238,6 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #else int result; #endif -#if defined(__APPLE__) && (defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT)) - int utime_dir_fd_unsupported = 0; -#endif utime_t utime; @@ -5285,17 +5354,8 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.13, *)) { -#endif result = utime_dir_fd(&utime, dir_fd, path->narrow, follow_symlinks); -#ifdef __APPLE__ - } else { - utime_dir_fd_unsupported = 1; - } -#endif - } else #endif @@ -5309,8 +5369,9 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, Py_END_ALLOW_THREADS -#if defined(__APPLE__) && (defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT)) - if (utime_dir_fd_unsupported) { +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + /* See utime_dir_fd implementation */ + if (result == -1 && errno == ENOSYS) { argument_unavailable_error(NULL, "dir_fd"); return NULL; } @@ -8332,25 +8393,30 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; ssize_t length; +#ifdef HAVE_READLINKAT + int readlinkat_unavailable = 0; +#endif Py_BEGIN_ALLOW_THREADS #ifdef HAVE_READLINKAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); -#ifdef __APPLE__ - } else { - argument_unavailable_error(NULL, "dir_fd"); - return NULL; - } -#endif + if (HAVE_READLINKAT_RUNTIME) { + length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); + } else { + readlinkat_unavailable = 1; + } } else #endif length = readlink(path->narrow, buffer, MAXPATHLEN); Py_END_ALLOW_THREADS +#ifdef HAVE_READLINKAT + if (readlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (length < 0) { return path_error(path); } @@ -8546,6 +8612,9 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, static int windows_has_symlink_unprivileged_flag = TRUE; #else int result; +#ifdef HAVE_SYMLINKAT + int symlinkat_unavailable = 0; +#endif #endif if (PySys_Audit("os.symlink", "OOi", src->object, dst->object, @@ -8608,23 +8677,25 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } Py_BEGIN_ALLOW_THREADS -#if HAVE_SYMLINKAT +#ifdef HAVE_SYMLINKAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - result = symlinkat(src->narrow, dir_fd, dst->narrow); -#ifdef __APPLE__ - } else { - argument_unavailable_error(NULL, "dir_fd"); - return NULL; - } -#endif + if (HAVE_SYMLINKAT_RUNTIME) { + result = symlinkat(src->narrow, dir_fd, dst->narrow); + } else { + symlinkat_unavailable = 1; + } } else #endif result = symlink(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#ifdef HAVE_SYMLINKAT + if (symlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif @@ -8920,17 +8991,13 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) #else #ifdef HAVE_OPENAT if (dir_fd != DEFAULT_DIR_FD) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif - fd = openat(dir_fd, path->narrow, flags, mode); + if (HAVE_OPENAT_RUNTIME) { + fd = openat(dir_fd, path->narrow, flags, mode); -#ifdef __APPLE__ - } else { - argument_unavailable_error(NULL, "dir_fd"); - return -1; - } -#endif + } else { + argument_unavailable_error(NULL, "dir_fd"); + return -1; + } } else #endif /* HAVE_OPENAT */ fd = open(path->narrow, flags, mode); @@ -13120,24 +13187,17 @@ _Py_COMP_DIAG_POP const char *path = PyBytes_AS_STRING(ub); if (self->dir_fd != DEFAULT_DIR_FD) { #ifdef HAVE_FSTATAT -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_FSTATAT_RUNTIME) { result = fstatat(self->dir_fd, path, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#ifdef __APPLE__ } else -#endif - -#endif -#if !defined(HAVE_FSTATAT) || defined(__APPLE__) +#endif /* HAVE_FSTATAT */ { Py_DECREF(ub); PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat"); return NULL; } -#endif /* HAVE_FSTATAT */ } else #endif @@ -13936,9 +13996,7 @@ os_scandir_impl(PyObject *module, path_t *path) errno = 0; #ifdef HAVE_FDOPENDIR if (iterator->path.fd != -1) { -#ifdef __APPLE__ - if (__builtin_available(macOS 10.10, *)) { -#endif + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(iterator->path.fd); if (fd == -1) @@ -13947,12 +14005,10 @@ os_scandir_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS iterator->dirp = fdopendir(fd); Py_END_ALLOW_THREADS -#ifdef __APPLE__ } else { argument_unavailable_error("opendir", "dir_fd"); return NULL; } -#endif } else #endif @@ -15336,12 +15392,9 @@ PyMODINIT_FUNC INITFUNC(void) { PyObject* module = PyModuleDef_Init(&posixmodule); -#if defined(__APPLE__) && defined(HAVE_PWRITEV) +#if defined(HAVE_PWRITEV) if (module) { - if (__builtin_available(macOS 11.0, *)) { - /* pass: ^^^^ cannot use '!' here */ - } else { - /* pwritev and preadv were introduced in macOS 11 */ + if (HAVE_PWRITEV_RUNTIME) {} else { PyObject* dct = PyModule_GetDict(module); if (dct == NULL) { diff --git a/Modules/timemodule.c b/Modules/timemodule.c index a97a6a18ec5d1d..b6db2d21cf5c2f 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -51,6 +51,12 @@ #define _Py_tzname tzname #endif +#if defined(__APPLE__ ) && defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +#else +# define HAVE_CLOCK_GETTIME_RUNTIME 1 +#endif + #define SEC_TO_NS (1000 * 1000 * 1000) /* Forward declarations */ @@ -1178,39 +1184,35 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) struct timespec ts; -#ifdef __APPLE__ - if (__builtin_available(macos 10.12, *)) { -#endif + if (HAVE_CLOCK_GETTIME_RUNTIME) { #ifdef CLOCK_PROF - const clockid_t clk_id = CLOCK_PROF; - const char *function = "clock_gettime(CLOCK_PROF)"; + const clockid_t clk_id = CLOCK_PROF; + const char *function = "clock_gettime(CLOCK_PROF)"; #else - const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; - const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; + const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; + const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; #endif - if (clock_gettime(clk_id, &ts) == 0) { - if (info) { - struct timespec res; - info->implementation = function; - info->monotonic = 1; - info->adjustable = 0; - if (clock_getres(clk_id, &res)) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; + if (clock_gettime(clk_id, &ts) == 0) { + if (info) { + struct timespec res; + info->implementation = function; + info->monotonic = 1; + info->adjustable = 0; + if (clock_getres(clk_id, &res)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; } - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - } - if (_PyTime_FromTimespec(tp, &ts) < 0) { - return -1; + if (_PyTime_FromTimespec(tp, &ts) < 0) { + return -1; + } + return 0; } - return 0; } -#ifdef __APPLE__ - } -#endif #endif /* getrusage(RUSAGE_SELF) */ @@ -1399,7 +1401,11 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #ifdef __APPLE__ static int -_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))); +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) + __attribute__((availability(macos, introduced=10.12))) + __attribute__((availability(ios, introduced=10.0))) + __attribute__((availability(tvos, introduced=10.0))) + __attribute__((availability(watchos, introduced=3.0))); #endif static int @@ -1528,16 +1534,16 @@ time_get_clock_info(PyObject *self, PyObject *args) else if (strcmp(name, "thread_time") == 0) { #ifdef __APPLE__ - if (__builtin_available(macos 10.12, *)) { + if (HAVE_CLOCK_GETTIME_RUNTIME) { #endif - if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + return NULL; + } +#ifdef __APPLE__ + } else { + PyErr_SetString(PyExc_ValueError, "unknown clock"); return NULL; } -#ifdef __APPLE__ - } else { - PyErr_SetString(PyExc_ValueError, "unknown clock"); - return NULL; - } #endif } #endif @@ -1825,107 +1831,71 @@ time_exec(PyObject *module) } #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) + if (HAVE_CLOCK_GETTIME_RUNTIME) { #ifdef CLOCK_REALTIME -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif - if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { - return -1; - } -#ifdef __APPLE__ - } -#endif + if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_MONOTONIC -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { - return -1; - } - -#ifdef __APPLE__ - } -#endif + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { + return -1; + } #endif #ifdef CLOCK_MONOTONIC_RAW -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { - return -1; - } -#ifdef __APPLE__ - } -#endif + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { + return -1; + } #endif #ifdef CLOCK_HIGHRES - if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { + return -1; + } #endif #ifdef CLOCK_PROCESS_CPUTIME_ID -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif - if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { - return -1; - } -#ifdef __APPLE__ - } -#endif + if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { + return -1; + } #endif #ifdef CLOCK_THREAD_CPUTIME_ID -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif - if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { - return -1; - } -#ifdef __APPLE__ - } -#endif + if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { + return -1; + } #endif #ifdef CLOCK_PROF - if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { + return -1; + } #endif #ifdef CLOCK_BOOTTIME - if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_TAI - if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME - if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME_RAW -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { + if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { + return -1; + } #endif - if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { - return -1; } -#ifdef __APPLE__ - } -#endif -#endif - #endif /* defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) */ if (!initialized) { @@ -1977,7 +1947,7 @@ PyInit_time(void) PyObject* module = PyModuleDef_Init(&timemodule); #if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) - if (__builtin_available(macOS 10.12, *)) { + if (HAVE_CLOCK_GETTIME_RUNTIME) { /* pass: ^^^ cannot use '!' here */ } else { PyObject* dct = PyModule_GetDict(module); @@ -2000,7 +1970,7 @@ PyInit_time(void) } #endif #if defined(__APPLE__) && defined(HAVE_THREAD_TIME) - if (__builtin_available(macOS 10.12, *)) { + if (HAVE_CLOCK_GETTIME_RUNTIME) { /* pass: ^^^ cannot use '!' here */ } else { PyObject* dct = PyModule_GetDict(module); diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index d10019e5b59b5d..47546e28a22cd0 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -25,6 +25,13 @@ # include #endif +#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME __builtin_available(macOS 10.12, iOS 10.10, tvOS 10.0, watchOS 3.0, *) +#else +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME 1 +#endif + + #ifdef Py_DEBUG int _Py_HashSecret_Initialized = 0; #else @@ -208,9 +215,14 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) error. getentropy() is retried if it failed with EINTR: interrupted by a signal. */ -#ifdef __APPLE__ + +#if defined(__APPLE__) && __has_attribute(availability) static int -py_getentropy(char *buffer, Py_ssize_t size, int raise) __attribute__((availability(macos,introduced=10.12))); +py_getentropy(char *buffer, Py_ssize_t size, int raise) + __attribute__((availability(macos,introduced=10.12))) + __attribute__((availability(ios,introduced=10.0))) + __attribute__((availability(tvos,introduced=10.0))) + __attribute__((availability(watchos,introduced=3.0))); #endif static int @@ -503,25 +515,21 @@ pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise) #else #if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) -#ifdef __APPLE__ - if (__builtin_available(macOS 10.12, *)) { -#endif + if (HAVE_GETENTRYPY_GETRANDOM_RUNTIME) { #ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, blocking, raise); + res = py_getrandom(buffer, size, blocking, raise); #else - res = py_getentropy(buffer, size, raise); -#endif - if (res < 0) { - return -1; - } - if (res == 1) { - return 0; - } - /* getrandom() or getentropy() function is not available: failed with - ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ -#ifdef __APPLE__ - } /* end of availability block */ + res = py_getentropy(buffer, size, raise); #endif + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; + } + /* getrandom() or getentropy() function is not available: failed with + ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ + } /* end of availability block */ #endif return dev_urandom(buffer, size, raise); diff --git a/Python/pytime.c b/Python/pytime.c index a1b49ff26ab294..2a7dfed5feb04b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -696,7 +696,7 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) #ifdef HAVE_CLOCK_GETTIME #if defined(__APPLE__) - if (__builtin_available(macOS 10.12, *)) { + if (__builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *)) { #endif err = clock_gettime(CLOCK_REALTIME, &ts); From 3a1d4f2c7bca89df4d57b528d8bd32771e82c9f5 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 20 Oct 2020 21:48:24 +0200 Subject: [PATCH 14/45] Dynamicly fill posix._have_functions as well --- Modules/posixmodule.c | 148 ++++++++++++++++++++++++++++++++---------- 1 file changed, 112 insertions(+), 36 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b329b21cfd41b9..d49e9d3d436e63 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15077,137 +15077,210 @@ all_ins(PyObject *m) } -static const char * const have_functions[] = { + +#define PROBE(name, test) \ + static int name(void) \ + { \ + if (test) { \ + return 1; \ + } else { \ + return 0; \ + } \ + } + +#ifdef HAVE_FSTATAT +PROBE(probe_fstatat, HAVE_FSTATAT_RUNTIME) +#endif #ifdef HAVE_FACCESSAT - "HAVE_FACCESSAT", +PROBE(probe_faccessat, HAVE_FACCESSAT_RUNTIME) +#endif + +#ifdef HAVE_FCHMODAT +PROBE(probe_fchmodat, HAVE_FCHMODAT_RUNTIME) +#endif + +#ifdef HAVE_FCHOWNAT +PROBE(probe_fchownat, HAVE_FCHOWNAT_RUNTIME) +#endif + +#ifdef HAVE_LINKAT +PROBE(probe_linkat, HAVE_LINKAT_RUNTIME) +#endif + +#ifdef HAVE_FDOPENDIR +PROBE(probe_fdopendir, HAVE_FDOPENDIR_RUNTIME) +#endif + +#ifdef HAVE_MKDIRAT +PROBE(probe_mkdirat, HAVE_MKDIRAT_RUNTIME) +#endif + +#ifdef HAVE_RENAMEAT +PROBE(probe_renameat, HAVE_RENAMEAT_RUNTIME) +#endif + +#ifdef HAVE_UNLINKAT +PROBE(probe_unlinkat, HAVE_UNLINKAT_RUNTIME) +#endif + +#ifdef HAVE_OPENAT +PROBE(probe_openat, HAVE_OPENAT_RUNTIME) +#endif + +#ifdef HAVE_READLINKAT +PROBE(probe_readlinkat, HAVE_READLINKAT_RUNTIME) +#endif + +#ifdef HAVE_SYMLINKAT +PROBE(probe_symlinkat, HAVE_SYMLINKAT_RUNTIME) +#endif + +#ifdef HAVE_FUTIMENS +PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) +#endif + +#ifdef HAVE_UTIMENSAT +PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) +#endif + + + + +static const struct have_function { + const char * const label; + int (*probe)(void); +} have_functions[] = { + +#ifdef HAVE_FACCESSAT + { "HAVE_FACCESSAT", probe_faccessat }, #endif #ifdef HAVE_FCHDIR - "HAVE_FCHDIR", + { "HAVE_FCHDIR", NULL }, #endif #ifdef HAVE_FCHMOD - "HAVE_FCHMOD", + { "HAVE_FCHMOD", NULL }, #endif #ifdef HAVE_FCHMODAT - "HAVE_FCHMODAT", + { "HAVE_FCHMODAT", probe_fchmodat }, #endif #ifdef HAVE_FCHOWN - "HAVE_FCHOWN", + { "HAVE_FCHOWN", NULL }, #endif #ifdef HAVE_FCHOWNAT - "HAVE_FCHOWNAT", + { "HAVE_FCHOWNAT", probe_fchownat }, #endif #ifdef HAVE_FEXECVE - "HAVE_FEXECVE", + { "HAVE_FEXECVE", NULL }, #endif #ifdef HAVE_FDOPENDIR - "HAVE_FDOPENDIR", + { "HAVE_FDOPENDIR", probe_fdopendir }, #endif #ifdef HAVE_FPATHCONF - "HAVE_FPATHCONF", + { "HAVE_FPATHCONF", NULL }, #endif #ifdef HAVE_FSTATAT - "HAVE_FSTATAT", + { "HAVE_FSTATAT", probe_fstatat }, #endif #ifdef HAVE_FSTATVFS - "HAVE_FSTATVFS", + { "HAVE_FSTATVFS", NULL }, #endif #if defined HAVE_FTRUNCATE || defined MS_WINDOWS - "HAVE_FTRUNCATE", + { "HAVE_FTRUNCATE", NULL }, #endif #ifdef HAVE_FUTIMENS - "HAVE_FUTIMENS", + { "HAVE_FUTIMENS", probe_futimens }, #endif #ifdef HAVE_FUTIMES - "HAVE_FUTIMES", + { "HAVE_FUTIMES", NULL }, #endif #ifdef HAVE_FUTIMESAT - "HAVE_FUTIMESAT", + { "HAVE_FUTIMESAT", probe_futimesat }, #endif #ifdef HAVE_LINKAT - "HAVE_LINKAT", + { "HAVE_LINKAT", probe_linkat }, #endif #ifdef HAVE_LCHFLAGS - "HAVE_LCHFLAGS", + { "HAVE_LCHFLAGS", NULL }, #endif #ifdef HAVE_LCHMOD - "HAVE_LCHMOD", + { "HAVE_LCHMOD", NULL }, #endif #ifdef HAVE_LCHOWN - "HAVE_LCHOWN", + { "HAVE_LCHOWN", NULL }, #endif #ifdef HAVE_LSTAT - "HAVE_LSTAT", + { "HAVE_LSTAT", NULL }, #endif #ifdef HAVE_LUTIMES - "HAVE_LUTIMES", + { "HAVE_LUTIMES", NULL }, #endif #ifdef HAVE_MEMFD_CREATE - "HAVE_MEMFD_CREATE", + { "HAVE_MEMFD_CREATE", NULL }, #endif #ifdef HAVE_MKDIRAT - "HAVE_MKDIRAT", + { "HAVE_MKDIRAT", probe_mkdirat }, #endif #ifdef HAVE_MKFIFOAT - "HAVE_MKFIFOAT", + { "HAVE_MKFIFOAT", NULL }, #endif #ifdef HAVE_MKNODAT - "HAVE_MKNODAT", + { "HAVE_MKNODAT", NULL }, #endif #ifdef HAVE_OPENAT - "HAVE_OPENAT", + { "HAVE_OPENAT", probe_openat }, #endif #ifdef HAVE_READLINKAT - "HAVE_READLINKAT", + { "HAVE_READLINKAT", probe_readlinkat }, #endif #ifdef HAVE_RENAMEAT - "HAVE_RENAMEAT", + { "HAVE_RENAMEAT", probe_renameat }, #endif #ifdef HAVE_SYMLINKAT - "HAVE_SYMLINKAT", + { "HAVE_SYMLINKAT", probe_symlinkat }, #endif #ifdef HAVE_UNLINKAT - "HAVE_UNLINKAT", + { "HAVE_UNLINKAT", probe_unlinkat }, #endif #ifdef HAVE_UTIMENSAT - "HAVE_UTIMENSAT", + { "HAVE_UTIMENSAT", probe_utimensat }, #endif #ifdef MS_WINDOWS - "MS_WINDOWS", + { "MS_WINDOWS", NULL }, #endif - NULL + { NULL, NULL } }; @@ -15357,14 +15430,17 @@ posixmodule_exec(PyObject *m) if (!list) { return -1; } - for (const char * const *trace = have_functions; *trace; trace++) { - PyObject *unicode = PyUnicode_DecodeASCII(*trace, strlen(*trace), NULL); + for (const struct have_function *trace = have_functions; trace->label; trace++) { + PyObject *unicode; + if (trace->probe && !trace->probe()) continue; + unicode = PyUnicode_DecodeASCII(trace->label, strlen(trace->label), NULL); if (!unicode) return -1; if (PyList_Append(list, unicode)) return -1; Py_DECREF(unicode); } + PyModule_AddObject(m, "_have_functions", list); return 0; From 2f019f4e25e5681c437cd60e0d48fbe8ab02efd3 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 01:37:31 -0700 Subject: [PATCH 15/45] Changes after testing with Xcode 6.2 on macOS 10.9 - __builtin_available is not present in the compiler, fall back to the ancient way of testing if a symbol is usable. - Logic error code in pytime.c resulted in a compile error. --- Modules/posixmodule.c | 58 ++++++++++++++++++++++++++++++++++++++++++- Python/pytime.c | 2 +- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d49e9d3d436e63..6aec1b2ba3da06 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -55,7 +55,9 @@ * In mixing the test with other tests or using negations will result in compile * errors. */ -#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) +#if defined(__APPLE__) + +#if defined(__has_builtin) && __has_builtin(__builtin_available) # define HAVE_FSTATAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) # define HAVE_FACCESSAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) # define HAVE_FCHMODAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) @@ -72,7 +74,60 @@ # define HAVE_UTIMENSAT_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) # define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +#else /* Xcode 8 or earlier */ + +#ifdef HAVE_FSTATAT +# define HAVE_FSTATAT_RUNTIME (fstatat != NULL) +#endif + +#ifdef HAVE_FACCESSAT +# define HAVE_FACCESSAT_RUNTIME (faccessat != NULL) +#endif + +#ifdef HAVE_FCHMODAT +# define HAVE_FCHMODAT_RUNTIME (fchmodat != NULL) +#endif + +#ifdef HAVE_FCHOWNAT +# define HAVE_FCHOWNAT_RUNTIME (fchownat != NULL) +#endif + +#ifdef HAVE_LINKAT +# define HAVE_LINKAT_RUNTIME (linkat != NULL) +#endif + +#ifdef HAVE_FDOPENDIR +# define HAVE_FDOPENDIR_RUNTIME (fdopendir != NULL) +#endif + +#ifdef HAVE_MKDIRAT +# define HAVE_MKDIRAT_RUNTIME (mkdirat != NULL) +#endif + +#ifdef HAVE_RENAMEAT +# define HAVE_RENAMEAT_RUNTIME (renameat != NULL) +#endif + +#ifdef HAVE_UNLINKAT +# define HAVE_UNLINKAT_RUNTIME (unlinkat != NULL) +#endif + +#ifdef HAVE_OPENAT +# define HAVE_OPENAT_RUNTIME (openat != NULL) +#endif + +#ifdef HAVE_READLINKAT +# define HAVE_READLINKAT_RUNTIME (readlinkat != NULL) +#endif + +#ifdef HAVE_SYMLINKAT +# define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) +#endif + +#endif + #ifdef HAVE_FUTIMESAT +/* Some of the logic for weak linking depends on this assertion */ # error "HAVE_FUTIMESAT unexpectedly defined" #endif @@ -94,6 +149,7 @@ # define HAVE_PWRITEV_RUNTIME 1 #endif + #ifdef __cplusplus extern "C" { #endif diff --git a/Python/pytime.c b/Python/pytime.c index 2a7dfed5feb04b..ede58eaf8a9c43 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -750,7 +750,7 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->adjustable = 1; } -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) } /* end of availibity block */ #endif From cec3da78308aa02b90f328d4dba5a5a050c5059f Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 10:54:49 +0200 Subject: [PATCH 16/45] Code formatting --- Modules/posixmodule.c | 80 ++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6aec1b2ba3da06..60b0b093ffe5e6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -76,53 +76,61 @@ #else /* Xcode 8 or earlier */ -#ifdef HAVE_FSTATAT -# define HAVE_FSTATAT_RUNTIME (fstatat != NULL) -#endif + /* __builtin_available is not present in these compilers, but + * some of the symbols might be weak linked (10.10 SDK or later + * deploying on 10.9. + * + * Fall back to the older style of availability checking for + * symbols introduced in macOS 10.10. + */ -#ifdef HAVE_FACCESSAT -# define HAVE_FACCESSAT_RUNTIME (faccessat != NULL) -#endif +# ifdef HAVE_FSTATAT +# define HAVE_FSTATAT_RUNTIME (fstatat != NULL) +# endif -#ifdef HAVE_FCHMODAT -# define HAVE_FCHMODAT_RUNTIME (fchmodat != NULL) -#endif +# ifdef HAVE_FACCESSAT +# define HAVE_FACCESSAT_RUNTIME (faccessat != NULL) +# endif -#ifdef HAVE_FCHOWNAT -# define HAVE_FCHOWNAT_RUNTIME (fchownat != NULL) -#endif +# ifdef HAVE_FCHMODAT +# define HAVE_FCHMODAT_RUNTIME (fchmodat != NULL) +# endif -#ifdef HAVE_LINKAT -# define HAVE_LINKAT_RUNTIME (linkat != NULL) -#endif +# ifdef HAVE_FCHOWNAT +# define HAVE_FCHOWNAT_RUNTIME (fchownat != NULL) +# endif -#ifdef HAVE_FDOPENDIR -# define HAVE_FDOPENDIR_RUNTIME (fdopendir != NULL) -#endif +# ifdef HAVE_LINKAT +# define HAVE_LINKAT_RUNTIME (linkat != NULL) +# endif -#ifdef HAVE_MKDIRAT -# define HAVE_MKDIRAT_RUNTIME (mkdirat != NULL) -#endif +# ifdef HAVE_FDOPENDIR +# define HAVE_FDOPENDIR_RUNTIME (fdopendir != NULL) +# endif -#ifdef HAVE_RENAMEAT -# define HAVE_RENAMEAT_RUNTIME (renameat != NULL) -#endif +# ifdef HAVE_MKDIRAT +# define HAVE_MKDIRAT_RUNTIME (mkdirat != NULL) +# endif -#ifdef HAVE_UNLINKAT -# define HAVE_UNLINKAT_RUNTIME (unlinkat != NULL) -#endif +# ifdef HAVE_RENAMEAT +# define HAVE_RENAMEAT_RUNTIME (renameat != NULL) +# endif -#ifdef HAVE_OPENAT -# define HAVE_OPENAT_RUNTIME (openat != NULL) -#endif +# ifdef HAVE_UNLINKAT +# define HAVE_UNLINKAT_RUNTIME (unlinkat != NULL) +# endif -#ifdef HAVE_READLINKAT -# define HAVE_READLINKAT_RUNTIME (readlinkat != NULL) -#endif +# ifdef HAVE_OPENAT +# define HAVE_OPENAT_RUNTIME (openat != NULL) +# endif -#ifdef HAVE_SYMLINKAT -# define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) -#endif +# ifdef HAVE_READLINKAT +# define HAVE_READLINKAT_RUNTIME (readlinkat != NULL) +# endif + +# ifdef HAVE_SYMLINKAT +# define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) +# endif #endif From 86b5cf3031b35d8c93c06ae5b810cc13ace21585 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 03:09:24 -0700 Subject: [PATCH 17/45] Fix a number of errors in weaklinking support --- Modules/posixmodule.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6aec1b2ba3da06..87262c3b79a626 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4093,8 +4093,8 @@ _posix_listdir(path_t *path, PyObject *list) dirp = fdopendir(fd); Py_END_ALLOW_THREADS } else { - /* XXX: This error message is suboptimal... */ - argument_unavailable_error(NULL, "path as file descriptor"); + PyErr_SetString(PyExc_TypeError, + "listdir: path should be string, bytes, os.PathLike or None, not int"); return NULL; } } @@ -4618,7 +4618,7 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is #ifdef HAVE_RENAMEAT if (renameat_unavailable) { - argument_unavailable_error(function_name, "dir_fd"); + argument_unavailable_error(function_name, "src_dir_fd and dst_dir_fd"); return NULL; } #endif @@ -4720,15 +4720,15 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) unlinkat_unavailable = 1; result = -1; } -#endif } else +#endif result = rmdir(path->narrow); #endif Py_END_ALLOW_THREADS -#ifdef HAVE_ULINKAT +#ifdef HAVE_UNLINKAT if (unlinkat_unavailable) { - argument_unavailable_error(NULL, "dir_fd"); + argument_unavailable_error("rmdir", "dir_fd"); return NULL; } #endif @@ -9022,6 +9022,9 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) { int fd; int async_err = 0; +#ifdef HAVE_OPENAT + int openat_unavailable = 0; +#endif #ifdef O_CLOEXEC int *atomic_flag_works = &_Py_open_cloexec_works; @@ -9051,8 +9054,8 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) fd = openat(dir_fd, path->narrow, flags, mode); } else { - argument_unavailable_error(NULL, "dir_fd"); - return -1; + openat_unavailable = 1; + fd = -1; } } else #endif /* HAVE_OPENAT */ @@ -9062,6 +9065,11 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); _Py_END_SUPPRESS_IPH + if (openat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return -1; + } + if (fd < 0) { if (!async_err) PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); @@ -14062,7 +14070,8 @@ os_scandir_impl(PyObject *module, path_t *path) iterator->dirp = fdopendir(fd); Py_END_ALLOW_THREADS } else { - argument_unavailable_error("opendir", "dir_fd"); + PyErr_SetString(PyExc_TypeError, + "scandir: path should be string, bytes, os.PathLike or None, not int"); return NULL; } } From dde0ba4ecc39c9fb422550fabcf614b31bdd1a91 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 12:10:33 +0200 Subject: [PATCH 18/45] Implement tests for weak linked symbols. The tests are not entirely done yet, and something similar needs to be done for the time module. --- Lib/test/test_posix.py | 214 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index be121ae463dbb3..ea4f694c622840 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1905,6 +1905,219 @@ def test_posix_spawnp(self): assert_python_ok(*args, PATH=path) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestPosixWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + def setUp(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] } + self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split(".")) + + def _verify_available(self, name): + if name not in self.available: + raise unittest.SkipTest(f"{name} not weak-linked") + + def test_pwritev(self): + self._verify_available("HAVE_PWRITEV") + if self.mac_ver >= (10, 16): + self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") + self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + + else: + self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") + self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + + def test_stat(self): + self._verify_available("HAVE_FSTATAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FSTATAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FSTATAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.stat("file", dir_fd=0) + + def test_access(self): + self._verify_available("HAVE_FACCESSAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FACCESSAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FACCESSAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.access("file", os.R_OK, dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.access("file", os.R_OK, follow_symlinks=False) + + with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"): + os.access("file", os.R_OK, effective_ids=True) + + def test_chmod(self): + self._verify_available("HAVE_FCHMODAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHMODAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHMODAT", posix._have_functions) + self.assertIn("HAVE_LCHMOD", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chmod("file", 0o644, dir_fd=0) + + def test_chown(self): + self._verify_available("HAVE_FCHOWNAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHOWNAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions) + self.assertIn("HAVE_LCHOWN", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chown("file", 0, 0, dir_fd=0) + + def test_link(self): + self._verify_available("HAVE_LINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_LINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_LINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): + os.link("source", "target", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) + + # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag + self.fail("Implement test") + + def test_listdir_scandir(self): + self._verify_available("HAVE_FDOPENDIR") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FDOPENDIR", posix._have_functions) + + else: + self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions) + + with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"): + os.listdir(0) + + with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"): + os.scandir(0) + + def test_mkdir(self): + self._verify_available("HAVE_MKDIRAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_MKDIRAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_MKDIRAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.mkdir("dir", dir_fd=0) + + def test_rename_replace(self): + self._verify_available("HAVE_RENAMEAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_RENAMEAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_RENAMEAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", dst_dir_fd=0) + + def test_unlink_rmdir(self): + self._verify_available("HAVE_UNLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_UNLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_UNLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.unlink("path", dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.rmdir("path", dir_fd=0) + + def test_open(self): + self._verify_available("HAVE_OPENAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_OPENAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_OPENAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.open("path", os.O_RDONLY, dir_fd=0) + + def test_readlink(self): + self._verify_available("HAVE_READLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_READLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_READLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.readlink("path", dir_fd=0) + + def test_symlink(self): + self._verify_available("HAVE_SYMLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_SYMLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.symlink("a", "b", dir_fd=0) + + def test_utime(self): + self._verify_available("HAVE_FUTIMENS_RUNTIME") + self._verify_available("HAVE_UTIMENSAT_RUNTIME") + if self.mac_ver >= (10, 13): + self.assertIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) + self.assertIn("HAVE_UTIMENSAT_RUNTIME, posix._have_functions) + + else: + self.assertNotIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) + self.assertNotIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.utime("path", dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.utime("path", follow_symlinks=False) + + # XXX: Check if this test is correct, the implementation of os.utime + # is fairly complicated + + + def test_main(): try: support.run_unittest( @@ -1912,6 +2125,7 @@ def test_main(): PosixGroupsTester, TestPosixSpawn, TestPosixSpawnP, + TestPosixWeaklinking ) finally: support.reap_children() From 7ac26c42c1742d5a9e67c03c962aebc89c63b60c Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 13:05:00 +0200 Subject: [PATCH 19/45] Finish the test for os.link behaviour --- Lib/test/test_posix.py | 46 +++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index ea4f694c622840..4175f5ccbc7f05 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1990,19 +1990,45 @@ def test_link(self): self.assertIn("HAVE_LINKAT", posix._have_functions) else: - self.assertNotIn("HAVE_LINKAT", posix._have_functions) + self.assertNotIn("HAVE_LINKAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): - os.link("source", "target", src_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): - os.link("source", "target", dst_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): + os.link("source", "target", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) + + # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag + base_path = os.path.abspath(support.TESTFN) + '.link' + link_path = os.path.join(base_path, "link") + target_path = os.path.join(base_path, "target") + source_path = os.path.join(base_path, "source") + try: + os.mkdir(base_path) + + with open(source_path, "w") as fp: + fp.write("data") + + os.symlink("target", link_path) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): - os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) + # Calling os.link should fail in the link(2) call, and + # should not reject *follow_symlinks* (to match the + # behaviour you'd get when building on a platform without + # linkat) + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=True) - # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag - self.fail("Implement test") + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=False) + + finally: + support.rmtree(base_path) + + + def test_listdir_scandir(self): self._verify_available("HAVE_FDOPENDIR") @@ -2101,7 +2127,7 @@ def test_utime(self): self._verify_available("HAVE_UTIMENSAT_RUNTIME") if self.mac_ver >= (10, 13): self.assertIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) - self.assertIn("HAVE_UTIMENSAT_RUNTIME, posix._have_functions) + self.assertIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) else: self.assertNotIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) From 191a2d78f776a0efcc4745dabc92ee72b92b519f Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 16:51:18 +0200 Subject: [PATCH 20/45] posixmodule.c uses multi-phase initialization --- Modules/posixmodule.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 61b30b0aa27fa8..4004ec68c006d9 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15362,6 +15362,23 @@ posixmodule_exec(PyObject *m) { _posixstate *state = get_posix_state(m); +#if defined(HAVE_PWRITEV) + if (HAVE_PWRITEV_RUNTIME) {} else { + PyObject* dct = PyModule_GetDict(m); + + if (dct == NULL) { + return NULL; + } + + if (PyDict_DelItemString(dct, "pwritev") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "preadv") == -1) { + PyErr_Clear(); + } + } +#endif + /* Initialize environ dictionary */ PyObject *v = convertenviron(); Py_XINCREF(v); @@ -15541,24 +15558,6 @@ PyMODINIT_FUNC INITFUNC(void) { PyObject* module = PyModuleDef_Init(&posixmodule); -#if defined(HAVE_PWRITEV) - if (module) { - if (HAVE_PWRITEV_RUNTIME) {} else { - PyObject* dct = PyModule_GetDict(module); - - if (dct == NULL) { - return NULL; - } - - if (PyDict_DelItemString(dct, "pwritev") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "preadv") == -1) { - PyErr_Clear(); - } - } - } -#endif return module; } From e6d195b7b078baeb3043607314e1dc8acf38a2ce Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 17:25:54 +0200 Subject: [PATCH 21/45] Make sure some form of prep_cif is called, even if the variadic version isn't present. I don't particularly like the code duplication, but this code should be pretty stable. --- Modules/_ctypes/callproc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 824b00af0e60c2..48a959c6b403c5 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -890,6 +890,16 @@ static int _call_function_pointer(int flags, "ffi_prep_cif_var failed"); return -1; } + } else { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } } else #endif From cfb02ba690eddeb1dfe629c0ad62741be46c86ae Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 17:49:37 +0200 Subject: [PATCH 22/45] Move the removal of unavailable functions to the module exec --- Modules/timemodule.c | 75 ++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index b6db2d21cf5c2f..83c8fa55f14ce4 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1825,6 +1825,43 @@ if it is -1, mktime() should guess based on the date and time.\n"); static int time_exec(PyObject *module) { +#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "clock_gettime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_getres") == -1) { + PyErr_Clear(); + } + } +#endif +#if defined(__APPLE__) && defined(HAVE_THREAD_TIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "thread_time") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { + PyErr_Clear(); + } + } +#endif /* Set, or reset, module variables like time.timezone */ if (init_timezone(module) < 0) { return -1; @@ -1946,44 +1983,6 @@ PyInit_time(void) { PyObject* module = PyModuleDef_Init(&timemodule); -#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) - if (HAVE_CLOCK_GETTIME_RUNTIME) { - /* pass: ^^^ cannot use '!' here */ - } else { - PyObject* dct = PyModule_GetDict(module); - - if (PyDict_DelItemString(dct, "clock_gettime") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "clock_settime") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "clock_getres") == -1) { - PyErr_Clear(); - } - } -#endif -#if defined(__APPLE__) && defined(HAVE_THREAD_TIME) - if (HAVE_CLOCK_GETTIME_RUNTIME) { - /* pass: ^^^ cannot use '!' here */ - } else { - PyObject* dct = PyModule_GetDict(module); - - if (PyDict_DelItemString(dct, "thread_time") == -1) { - PyErr_Clear(); - } - if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { - PyErr_Clear(); - } - } -#endif - return module; } From 003dae8966170974c1f9c9070d4c1706d87c41bd Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 19:13:23 +0200 Subject: [PATCH 23/45] Improved error handling --- Modules/posixmodule.c | 2 +- Modules/timemodule.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4004ec68c006d9..83e30d7f607b4c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15367,7 +15367,7 @@ posixmodule_exec(PyObject *m) PyObject* dct = PyModule_GetDict(m); if (dct == NULL) { - return NULL; + return -1; } if (PyDict_DelItemString(dct, "pwritev") == -1) { diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 83c8fa55f14ce4..dcffbb3d1defff 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1830,6 +1830,9 @@ time_exec(PyObject *module) /* pass: ^^^ cannot use '!' here */ } else { PyObject* dct = PyModule_GetDict(module); + if (dct == NULL) { + return -1; + } if (PyDict_DelItemString(dct, "clock_gettime") == -1) { PyErr_Clear(); From 004ba4ef0747c022e795a8040af473819edfc113 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Wed, 21 Oct 2020 20:39:53 +0200 Subject: [PATCH 24/45] Remove '-arch arm64' from the compiler arguments when necessary This patch is not 100% correct: It ignores Xcode 12.2 on macOS 10.15. --- Lib/_osx_support.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 8ed1eeac8525da..53a3b59dd24b14 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -140,6 +140,20 @@ def _supports_universal_builds(): osx_version = '' return bool(osx_version >= (10, 4)) if osx_version else False +def _supports_arm64_builds(): + """Returns jTrue if arm64 buids are supported on this system""" + # There are two sets of systems supporting macOS/arm64 builds: + # 1. macOS 11 and later, unconditionally + # 2. macOS 10.15 with Xcode 12.2 or later + # For now the second category is ignored. + osx_version = _get_system_version() + if osx_version: + try: + osx_version = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + osx_version = '' + return osx_version >= (11, 0) if osx_version else False + def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" @@ -331,6 +345,13 @@ def compiler_fixup(compiler_so, cc_args): except ValueError: break + elif not _supports_arm64_builds(): + # Look for "-arch arm64" and drop that + for idx in range(len(compiler_so)): + if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": + del compiler_so[idx:idx+2] + break + if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig From 6019346c3020e516f30d18f0a77eb878239474b3 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 11:02:23 +0200 Subject: [PATCH 25/45] check that weak linking works for the time module --- Lib/test/test_time.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6ced0470d07561..6f7fc7869710bd 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1041,6 +1041,36 @@ def test_object_to_timespec(self): with self.assertRaises(ValueError): pytime_object_to_timespec(float('nan'), time_rnd) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestTimeWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + def setUp(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] } + self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split(".")) + + def _verify_available(self, name): + if name not in self.available: + raise unittest.SkipTest(f"{name} not weak-linked") + + def test_clock_functions(self): + self._verify_available("HAVE_CLOCK_GETTIME") + clock_names = [ + "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", + "clock_settime_ns", "clock_getres"] + if self.mac_ver >= (10, 16): + for name in clock_names: + self.assertTrue(hasattr(time, name), "time.{name} is not available") + + else: + for name in clock_names: + self.assertFalse(hasattr(time, name), "time.{name} is available") + if __name__ == "__main__": unittest.main() From 54576ab20b5d6ebc33a8228b2d56313151dfd77d Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 14:56:46 +0200 Subject: [PATCH 26/45] Drop the DEPS_ONLY changes This change is independent of the macOS 11/arm64 changes. --- Mac/BuildScript/build-installer.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 56fc20390c3f97..5b56c0687c275c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -150,10 +150,6 @@ def getFullVersion(): # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level DEPTARGET = '10.5' -# If true only builds the 3th-party dependencies -# in $WORKDIR -DEPS_ONLY=False - def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) @@ -649,7 +645,6 @@ def parseOptions(args=None): global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY - global DEPS_ONLY if args is None: args = sys.argv[1:] @@ -657,7 +652,7 @@ def parseOptions(args=None): try: options, args = getopt.getopt(args, '?hb', [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', - 'dep-target=', 'universal-archs=', 'deps-only', 'help' ]) + 'dep-target=', 'universal-archs=', 'help' ]) except getopt.GetoptError: print(sys.exc_info()[1]) sys.exit(1) @@ -678,9 +673,6 @@ def parseOptions(args=None): elif k in ('--third-party',): DEPSRC=v - elif k in ('--deps-only',): - DEPS_ONLY=True - elif k in ('--sdk-path',): print(" WARNING: --sdk-path is no longer supported") @@ -726,8 +718,6 @@ def parseOptions(args=None): print(" -- Building a Python %s framework at patch level %s" % (getVersion(), getFullVersion())) print("") - if DEPS_ONLY: - print("Stopping after building third-party libraries") def extractArchive(builddir, archiveName): """ @@ -1694,9 +1684,6 @@ def main(): # Then build third-party libraries such as sleepycat DB4. buildLibraries() - if DEPS_ONLY: - sys.exit(1) - # Now build python itself buildPython() From 6af77ab3d4799c8190f83a473a4271f2bbae3cc1 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 14:59:46 +0200 Subject: [PATCH 27/45] Undo change to default PATH --- Mac/BuildScript/build-installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5b56c0687c275c..0315c39385d76c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -623,7 +623,7 @@ def checkEnvironment(): ev, os.environ[ev])) del os.environ[ev] - base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' + base_path = '/bin:/sbin:/usr/bin:/usr/sbin' if 'SDK_TOOLS_BIN' in os.environ: base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; From b653df999473121daf901dd88cbff5d7252bfc96 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 15:32:47 +0200 Subject: [PATCH 28/45] Use configure to detect the presence of _dyld_shared_cache_contains_path --- Modules/_ctypes/callproc.c | 58 +++++++++++++++++--------------------- configure | 25 ++++++++++++++++ configure.ac | 6 ++++ pyconfig.h.in | 3 ++ 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 48a959c6b403c5..09f97d682af7ed 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -67,15 +67,7 @@ #endif #ifdef __APPLE__ -/* - * The API to query if a shared library is in the shared cache is - * private for now, this should change in beta 4. - * - * TODO: - * - Switch to that API - * - Add feature macro and runtime guards (as with the ffi.*loc API's) - */ -extern bool _dyld_shared_cache_contains_path(const char* path) __attribute__((weak_import)); +#include #endif #ifdef MS_WIN32 @@ -1468,36 +1460,38 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else -#ifdef __APPLE__ - static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) - { +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) +{ PyObject *name, *name2; char *name_str; - if (_dyld_shared_cache_contains_path == NULL) { + if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + if (PyBytes_Check(name2)) + name_str = PyBytes_AS_STRING(name2); + else + name_str = PyByteArray_AS_STRING(name2); + + if(_dyld_shared_cache_contains_path(name_str)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + + } else { PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); return NULL; } - if (!PyArg_ParseTuple(args, "O", &name)) - return NULL; - - if (name == Py_None) - Py_RETURN_FALSE; - - if (PyUnicode_FSConverter(name, &name2) == 0) - return NULL; - if (PyBytes_Check(name2)) - name_str = PyBytes_AS_STRING(name2); - else - name_str = PyByteArray_AS_STRING(name2); - - if(_dyld_shared_cache_contains_path(name_str)) - Py_RETURN_TRUE; - else - Py_RETURN_FALSE; } - #endif +#endif static PyObject *py_dl_open(PyObject *self, PyObject *args) { @@ -2012,7 +2006,7 @@ PyMethodDef _ctypes_module_methods[] = { {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, #endif -#ifdef __APPLE__ +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, diff --git a/configure b/configure index f811d2c23e4a7a..b5467fc862c131 100755 --- a/configure +++ b/configure @@ -11983,6 +11983,31 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _dyld_shared_cache_contains_path" >&5 +$as_echo_n "checking for _dyld_shared_cache_contains_path... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +void *x=_dyld_shared_cache_contains_path + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index d2044d0520e679..7f3bf532806eeb 100644 --- a/configure.ac +++ b/configure.ac @@ -3766,6 +3766,12 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no) ]) +AC_MSG_CHECKING(for _dyld_shared_cache_contains_path) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[void *x=_dyld_shared_cache_contains_path]])], + [AC_DEFINE(HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH, 1, Define if you have the '_dyld_shared_cache_contains_path' function.) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) +]) AC_MSG_CHECKING(for memfd_create) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/pyconfig.h.in b/pyconfig.h.in index bc906a869b623e..492383400b5d3f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -781,6 +781,9 @@ /* Define if you have the 'prlimit' functions. */ #undef HAVE_PRLIMIT +/* Define if you have the '_dyld_shared_cache_contains_path' function. */ +#undef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH + /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H From 98af7b3db53da3c5b1aa3e6bd3aac9404b31835e Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 15:34:12 +0200 Subject: [PATCH 29/45] Revert unnecessary change --- Modules/posixmodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 83e30d7f607b4c..c7736219da134f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15557,9 +15557,7 @@ static struct PyModuleDef posixmodule = { PyMODINIT_FUNC INITFUNC(void) { - PyObject* module = PyModuleDef_Init(&posixmodule); - - return module; + return PyModuleDef_Init(&posixmodule); } #ifdef __cplusplus From 817d9bfbc562b370ae17353adfc32261bfb479f9 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 15:37:12 +0200 Subject: [PATCH 30/45] Updates to timemodule.c - Check availability of __attribute__((availability - Revert unneccessary change to module init function --- Modules/timemodule.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index dcffbb3d1defff..cf7ca10bf5ec25 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1399,7 +1399,7 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) #define HAVE_THREAD_TIME -#ifdef __APPLE__ +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) static int _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) __attribute__((availability(macos, introduced=10.12))) @@ -1984,9 +1984,7 @@ static struct PyModuleDef timemodule = { PyMODINIT_FUNC PyInit_time(void) { - PyObject* module = PyModuleDef_Init(&timemodule); - - return module; + return PyModuleDef_Init(&timemodule); } /* Implement pysleep() for various platforms. From 8684d9de43878b3c5690d6e0bec1fb4ac0968ea1 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 15:44:59 +0200 Subject: [PATCH 31/45] Stricter checking of __builtin_available --- Python/bootstrap_hash.c | 2 +- Python/pytime.c | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 47546e28a22cd0..11e407e778453d 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -216,7 +216,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) getentropy() is retried if it failed with EINTR: interrupted by a signal. */ -#if defined(__APPLE__) && __has_attribute(availability) +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) static int py_getentropy(char *buffer, Py_ssize_t size, int raise) __attribute__((availability(macos,introduced=10.12))) diff --git a/Python/pytime.c b/Python/pytime.c index ede58eaf8a9c43..7325dca0150a81 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -5,6 +5,10 @@ #if defined(__APPLE__) #include /* mach_absolute_time(), mach_timebase_info() */ + +#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +#endif #endif #define _PyTime_check_mul_overflow(a, b) \ @@ -695,8 +699,8 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) #ifdef HAVE_CLOCK_GETTIME -#if defined(__APPLE__) - if (__builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *)) { +#ifdef HAVE_CLOCK_GETTIME_RUNTIME + if (HAVE_CLOCK_GETTIME_RUNTIME) { #endif err = clock_gettime(CLOCK_REALTIME, &ts); @@ -723,13 +727,13 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) } } -#ifdef __APPLE__ +#ifdef HAVE_CLOCK_GETTIME_RUNTIME } else { #endif #endif -#if !defined(HAVE_CLOCK_GETTIME) || defined(__APPLE__) +#if !defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_GETTIME_RUNTIME) /* test gettimeofday() */ err = gettimeofday(&tv, (struct timezone *)NULL); @@ -750,7 +754,7 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->adjustable = 1; } -#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) +#if defined(HAVE_CLOCK_GETTIME_RUNTIME) && defined(HAVE_CLOCK_GETTIME) } /* end of availibity block */ #endif From 0b4461084e80c51528ba3c91378572a43df06f47 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 15:53:54 +0200 Subject: [PATCH 32/45] Clean up setup.py --- setup.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 3bbf17a1c2283e..731eaad34d295f 100644 --- a/setup.py +++ b/setup.py @@ -229,13 +229,6 @@ def macosx_sdk_specified(): macosx_sdk_root() return MACOS_SDK_SPECIFIED -def is_macosx_at_least(vers): - if MACOS: - dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') - if dep_target: - return tuple(map(int, dep_target.split('.'))) >= vers - return False - def is_macosx_sdk_path(path): """ @@ -2120,8 +2113,7 @@ def configure_ctypes(self, ext): def detect_ctypes(self): # Thomas Heller's _ctypes module - if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and - (is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS): self.use_system_libffi = True else: self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") @@ -2139,7 +2131,6 @@ def detect_ctypes(self): if MACOS: sources.append('_ctypes/malloc_closure.c') extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') - #sources.append('_ctypes/darwin/dlfcn_simple.c') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') From 36deb92813ba489662d3c56176ef4e957156b3f2 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 17:32:39 +0200 Subject: [PATCH 33/45] Fix build failures on Linux --- Modules/posixmodule.c | 5 +++-- Python/bootstrap_hash.c | 9 ++++++--- Python/pytime.c | 6 ++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c7736219da134f..ff31f60d171314 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3939,8 +3939,9 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, dst_dir_fd, dst->narrow, follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + } #ifdef __APPLE__ - } else { + else { if (src_dir_fd == DEFAULT_DIR_FD && dst_dir_fd == DEFAULT_DIR_FD) { /* See issue 41355: This matches the behaviour of !HAVE_LINKAT */ result = link(src->narrow, dst->narrow); @@ -3948,8 +3949,8 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, linkat_unavailable = 1; } } - } #endif + } else #endif /* HAVE_LINKAT */ result = link(src->narrow, dst->narrow); diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 11e407e778453d..9f9a5a363b0a31 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -25,9 +25,12 @@ # include #endif -#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) -# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME __builtin_available(macOS 10.12, iOS 10.10, tvOS 10.0, watchOS 3.0, *) -#else +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME __builtin_available(macOS 10.12, iOS 10.10, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_GETENTRYPY_GETRANDOM_RUNTIME # define HAVE_GETENTRYPY_GETRANDOM_RUNTIME 1 #endif diff --git a/Python/pytime.c b/Python/pytime.c index 7325dca0150a81..89d63e080422b3 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -6,8 +6,10 @@ #if defined(__APPLE__) #include /* mach_absolute_time(), mach_timebase_info() */ -#if defined(__APPLE__) && defined(__has_builtin) && __has_builtin(__builtin_available) -# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif #endif #endif From 33d5710b1ea435a4ffe65bf95fcf8b7b177ef8bc Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 17:37:40 +0200 Subject: [PATCH 34/45] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł Piotr Przeradowski --- Lib/_osx_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 53a3b59dd24b14..89b736224a9ceb 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -141,7 +141,7 @@ def _supports_universal_builds(): return bool(osx_version >= (10, 4)) if osx_version else False def _supports_arm64_builds(): - """Returns jTrue if arm64 buids are supported on this system""" + """Returns True if arm64 builds are supported on this system""" # There are two sets of systems supporting macOS/arm64 builds: # 1. macOS 11 and later, unconditionally # 2. macOS 10.15 with Xcode 12.2 or later From c3113ebeb67c81705c427a0632e68a94723d6c62 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 19:09:51 +0200 Subject: [PATCH 35/45] Remove reference to non-existing probe function --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ff31f60d171314..b69345eac1538c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15283,7 +15283,7 @@ static const struct have_function { #endif #ifdef HAVE_FUTIMESAT - { "HAVE_FUTIMESAT", probe_futimesat }, + { "HAVE_FUTIMESAT", NULL }, #endif #ifdef HAVE_LINKAT From 587e53e5b3f210e7473330b922950544cbdbd9de Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 20:52:59 +0200 Subject: [PATCH 36/45] One more attempt at building on Linux... --- Modules/timemodule.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modules/timemodule.c b/Modules/timemodule.c index cf7ca10bf5ec25..2faf5792be3a73 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -51,9 +51,12 @@ #define _Py_tzname tzname #endif -#if defined(__APPLE__ ) && defined(__has_builtin) && __has_builtin(__builtin_available) -# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) -#else +#if defined(__APPLE__ ) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_CLOCK_GETTIME_RUNTIME # define HAVE_CLOCK_GETTIME_RUNTIME 1 #endif From 24ef276ff5a5895facfc2290e1fb700e7608c171 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 20:54:48 +0200 Subject: [PATCH 37/45] A bit of clean up of the setup.py logic. - ffi_inc was unconditionally set to the SDK path (even if LIBFFI_INCLUDEDIR was set) - USING_APPLE_OS_LIBFFI should only be set when actually using the SDK version --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 731eaad34d295f..b0d43afe349fca 100644 --- a/setup.py +++ b/setup.py @@ -2167,17 +2167,17 @@ def detect_ctypes(self): ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - # XXX: The define should only be added when actually using the system - # version (and not a locally compiled one) - ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") - if os.path.exists(ffi_in_sdk): - ffi_inc = ffi_in_sdk - ffi_lib = 'ffi' - else: - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + if os.path.exists(ffi_in_sdk): + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') if not ffi_inc: found = find_file('ffi.h', [], ffi_inc_dirs) From e0614bc254c6ee00b9942069eb0a75de2883ac10 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 21:32:11 +0200 Subject: [PATCH 38/45] Fix patchcheck issues --- Lib/ctypes/macholib/dyld.py | 8 +- Lib/test/test_posix.py | 198 ++++++++++++++--------------- Lib/test/test_time.py | 10 +- Mac/BuildScript/build-installer.py | 8 +- Modules/posixmodule.c | 4 +- Python/bootstrap_hash.c | 8 +- setup.py | 2 +- 7 files changed, 119 insertions(+), 119 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 22825d0d15b290..1c3f8fd38b0665 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -7,10 +7,10 @@ from ctypes.macholib.dylib import dylib_info from itertools import * try: - from _ctypes import _dyld_shared_cache_contains_path + from _ctypes import _dyld_shared_cache_contains_path except ImportError: - def _dyld_shared_cache_contains_path(*args): - raise NotImplementedError + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -134,7 +134,7 @@ def dyld_find(name, executable_path=None, env=None): if _dyld_shared_cache_contains_path(path): return path except NotImplementedError: - pass + pass raise ValueError("dylib %s could not be found" % (name,)) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 4175f5ccbc7f05..117d67775f88e3 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1907,7 +1907,7 @@ def test_posix_spawnp(self): @unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") class TestPosixWeaklinking(unittest.TestCase): - # These test cases verify that weak linking support on macOS works + # These test cases verify that weak linking support on macOS works # as expected. These cases only test new behaviour introduced by weak linking, # regular behaviour is tested by the normal test cases. def setUp(self): @@ -1925,69 +1925,69 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") + self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") + self.assertFalse(hasattr(os, "preadv"), "os.readv is available") def test_stat(self): self._verify_available("HAVE_FSTATAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_FSTATAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FSTATAT", posix._have_functions) else: - self.assertNotIn("HAVE_FSTATAT", posix._have_functions) + self.assertNotIn("HAVE_FSTATAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.stat("file", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.stat("file", dir_fd=0) def test_access(self): self._verify_available("HAVE_FACCESSAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_FACCESSAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FACCESSAT", posix._have_functions) else: - self.assertNotIn("HAVE_FACCESSAT", posix._have_functions) + self.assertNotIn("HAVE_FACCESSAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.access("file", os.R_OK, dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.access("file", os.R_OK, dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): - os.access("file", os.R_OK, follow_symlinks=False) + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.access("file", os.R_OK, follow_symlinks=False) - with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"): - os.access("file", os.R_OK, effective_ids=True) + with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"): + os.access("file", os.R_OK, effective_ids=True) def test_chmod(self): self._verify_available("HAVE_FCHMODAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_FCHMODAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHMODAT", posix._have_functions) else: - self.assertNotIn("HAVE_FCHMODAT", posix._have_functions) - self.assertIn("HAVE_LCHMOD", posix._have_functions) + self.assertNotIn("HAVE_FCHMODAT", posix._have_functions) + self.assertIn("HAVE_LCHMOD", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.chmod("file", 0o644, dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chmod("file", 0o644, dir_fd=0) def test_chown(self): self._verify_available("HAVE_FCHOWNAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_FCHOWNAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHOWNAT", posix._have_functions) else: - self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions) - self.assertIn("HAVE_LCHOWN", posix._have_functions) + self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions) + self.assertIn("HAVE_LCHOWN", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.chown("file", 0, 0, dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chown("file", 0, 0, dir_fd=0) def test_link(self): self._verify_available("HAVE_LINKAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_LINKAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_LINKAT", posix._have_functions) else: self.assertNotIn("HAVE_LINKAT", posix._have_functions) @@ -1997,10 +1997,10 @@ def test_link(self): with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): os.link("source", "target", dst_dir_fd=0) - + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) - + # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag base_path = os.path.abspath(support.TESTFN) + '.link' link_path = os.path.join(base_path, "link") @@ -2008,14 +2008,14 @@ def test_link(self): source_path = os.path.join(base_path, "source") try: os.mkdir(base_path) - + with open(source_path, "w") as fp: fp.write("data") os.symlink("target", link_path) # Calling os.link should fail in the link(2) call, and - # should not reject *follow_symlinks* (to match the + # should not reject *follow_symlinks* (to match the # behaviour you'd get when building on a platform without # linkat) with self.assertRaises(FileExistsError): @@ -2026,122 +2026,122 @@ def test_link(self): finally: support.rmtree(base_path) - - - + + + def test_listdir_scandir(self): self._verify_available("HAVE_FDOPENDIR") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_FDOPENDIR", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FDOPENDIR", posix._have_functions) else: - self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions) + self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions) + + with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"): + os.listdir(0) - with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"): - os.listdir(0) + with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"): + os.scandir(0) - with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"): - os.scandir(0) - def test_mkdir(self): self._verify_available("HAVE_MKDIRAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_MKDIRAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_MKDIRAT", posix._have_functions) else: - self.assertNotIn("HAVE_MKDIRAT", posix._have_functions) + self.assertNotIn("HAVE_MKDIRAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.mkdir("dir", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.mkdir("dir", dir_fd=0) def test_rename_replace(self): self._verify_available("HAVE_RENAMEAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_RENAMEAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_RENAMEAT", posix._have_functions) else: - self.assertNotIn("HAVE_RENAMEAT", posix._have_functions) + self.assertNotIn("HAVE_RENAMEAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): - os.rename("a", "b", src_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", src_dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): - os.rename("a", "b", dst_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", dst_dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): - os.replace("a", "b", src_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", src_dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): - os.replace("a", "b", dst_dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", dst_dir_fd=0) def test_unlink_rmdir(self): self._verify_available("HAVE_UNLINKAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_UNLINKAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_UNLINKAT", posix._have_functions) else: - self.assertNotIn("HAVE_UNLINKAT", posix._have_functions) + self.assertNotIn("HAVE_UNLINKAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.unlink("path", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.unlink("path", dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.rmdir("path", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.rmdir("path", dir_fd=0) def test_open(self): self._verify_available("HAVE_OPENAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_OPENAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_OPENAT", posix._have_functions) else: - self.assertNotIn("HAVE_OPENAT", posix._have_functions) + self.assertNotIn("HAVE_OPENAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.open("path", os.O_RDONLY, dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.open("path", os.O_RDONLY, dir_fd=0) - def test_readlink(self): self._verify_available("HAVE_READLINKAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_READLINKAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_READLINKAT", posix._have_functions) else: - self.assertNotIn("HAVE_READLINKAT", posix._have_functions) + self.assertNotIn("HAVE_READLINKAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.readlink("path", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.readlink("path", dir_fd=0) def test_symlink(self): self._verify_available("HAVE_SYMLINKAT") - if self.mac_ver >= (10, 10): - self.assertIn("HAVE_SYMLINKAT", posix._have_functions) + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_SYMLINKAT", posix._have_functions) else: - self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions) + self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.symlink("a", "b", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.symlink("a", "b", dir_fd=0) def test_utime(self): self._verify_available("HAVE_FUTIMENS_RUNTIME") self._verify_available("HAVE_UTIMENSAT_RUNTIME") - if self.mac_ver >= (10, 13): - self.assertIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) - self.assertIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + if self.mac_ver >= (10, 13): + self.assertIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) + self.assertIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) else: - self.assertNotIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) - self.assertNotIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + self.assertNotIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) + self.assertNotIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.utime("path", dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): - os.utime("path", dir_fd=0) + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.utime("path", follow_symlinks=False) - with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): - os.utime("path", follow_symlinks=False) + # XXX: Check if this test is correct, the implementation of os.utime + # is fairly complicated - # XXX: Check if this test is correct, the implementation of os.utime - # is fairly complicated - def test_main(): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6f7fc7869710bd..9184ce02b9e58f 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1061,15 +1061,15 @@ def _verify_available(self, name): def test_clock_functions(self): self._verify_available("HAVE_CLOCK_GETTIME") clock_names = [ - "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", + "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", "clock_settime_ns", "clock_getres"] if self.mac_ver >= (10, 16): - for name in clock_names: - self.assertTrue(hasattr(time, name), "time.{name} is not available") + for name in clock_names: + self.assertTrue(hasattr(time, name), "time.{name} is not available") else: - for name in clock_names: - self.assertFalse(hasattr(time, name), "time.{name} is available") + for name in clock_names: + self.assertFalse(hasattr(time, name), "time.{name} is available") if __name__ == "__main__": diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0315c39385d76c..2e8ce9f9101d88 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -197,9 +197,9 @@ def tweak_tcl_build(basedir, archList): with open("Makefile", "r") as fp: contents = fp.readlines() - # For reasons I don't understand the tcl configure script - # decides that some stdlib symbols aren't present, before - # deciding that strtod is broken. + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. new_contents = [] for line in contents: if line.startswith("COMPAT_OBJS"): @@ -209,7 +209,7 @@ def tweak_tcl_build(basedir, archList): for nm in ("strstr.o", "strtoul.o", " strtod.o"): line = line.replace(nm, "") new_contents.append(line) - + with open("Makefile", "w") as fp: fp.writelines(new_contents) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b69345eac1538c..ba38145d849002 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5137,7 +5137,7 @@ utime_fd(utime_t *ut, int fd) } else #ifndef HAVE_FUTIMES { - /* Not sure if this can happen */ + /* Not sure if this can happen */ PyErr_SetString( PyExc_RuntimeError, "neither futimens nor futimes are supported" @@ -5176,7 +5176,7 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) } else #ifndef HAVE_LUTIMES { - /* Not sure if this can happen */ + /* Not sure if this can happen */ PyErr_SetString( PyExc_RuntimeError, "neither utimensat nor lutimes are supported" diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 9f9a5a363b0a31..a212f69870ed10 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -222,10 +222,10 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) #if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) static int py_getentropy(char *buffer, Py_ssize_t size, int raise) - __attribute__((availability(macos,introduced=10.12))) - __attribute__((availability(ios,introduced=10.0))) - __attribute__((availability(tvos,introduced=10.0))) - __attribute__((availability(watchos,introduced=3.0))); + __attribute__((availability(macos,introduced=10.12))) + __attribute__((availability(ios,introduced=10.0))) + __attribute__((availability(tvos,introduced=10.0))) + __attribute__((availability(watchos,introduced=3.0))); #endif static int diff --git a/setup.py b/setup.py index b0d43afe349fca..14431b5fa4d413 100644 --- a/setup.py +++ b/setup.py @@ -2202,7 +2202,7 @@ def detect_ctypes(self): ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") if grep_headers_for('ffi_closure_alloc', ffi_headers): ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") - + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From e6478faa6940a50e9c228371890304ebcb26c1e8 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 21:33:22 +0200 Subject: [PATCH 39/45] add prefix to f-strings --- Lib/test/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 9184ce02b9e58f..5b6a2e592d3754 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1065,11 +1065,11 @@ def test_clock_functions(self): "clock_settime_ns", "clock_getres"] if self.mac_ver >= (10, 16): for name in clock_names: - self.assertTrue(hasattr(time, name), "time.{name} is not available") + self.assertTrue(hasattr(time, name), f"time.{name} is not available") else: for name in clock_names: - self.assertFalse(hasattr(time, name), "time.{name} is available") + self.assertFalse(hasattr(time, name), f"time.{name} is available") if __name__ == "__main__": From 3f72949f83da0e56e87d360c70a66804f1e5ef41 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 21:41:23 +0200 Subject: [PATCH 40/45] Add missing preprocessor guards --- Modules/posixmodule.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ba38145d849002..63d5aaea8f81ef 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2521,10 +2521,12 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, #endif /* MS_WINDOWS */ Py_END_ALLOW_THREADS +#ifdef HAVE_FSTATAT if (fstatat_unavailable) { argument_unavailable_error("stat", "dir_fd"); return NULL; } +#endif if (result != 0) { return path_error(path); @@ -9074,10 +9076,12 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); _Py_END_SUPPRESS_IPH +#ifdef HAVE_OPENAT if (openat_unavailable) { argument_unavailable_error(NULL, "dir_fd"); return -1; } +#endif if (fd < 0) { if (!async_err) From 5daff99935f3212b6907a2da3b2facd44ffa9f4d Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 21:56:54 +0200 Subject: [PATCH 41/45] Use the correct cut-off for the availability of clock_gettime and friends --- Lib/test/test_time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 5b6a2e592d3754..303bdf90aea05b 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1063,7 +1063,7 @@ def test_clock_functions(self): clock_names = [ "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", "clock_settime_ns", "clock_getres"] - if self.mac_ver >= (10, 16): + if self.mac_ver >= (10, 12): for name in clock_names: self.assertTrue(hasattr(time, name), f"time.{name} is not available") From 668126168391567c0e707fbf85c3cdcf7fb00079 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 22 Oct 2020 22:21:07 +0200 Subject: [PATCH 42/45] Fix error in preprocessor logic for os.chmod implementation --- Modules/posixmodule.c | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 63d5aaea8f81ef..4b1724ece63778 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -42,7 +42,7 @@ /* * A number of APIs are available on macOS from a certain macOS version. * To support building with a new SDK while deploying to older versions - * the availability test is split into two: + * the availability test is split into two: * - HAVE_: The configure check for compile time availability * - HAVE__RUNTIME: Runtime check for availability * @@ -52,10 +52,10 @@ * Due to compiler restrictions there is one valid use of HAVE__RUNTIME: * if (HAVE__RUNTIME) { ... } * - * In mixing the test with other tests or using negations will result in compile + * In mixing the test with other tests or using negations will result in compile * errors. */ -#if defined(__APPLE__) +#if defined(__APPLE__) #if defined(__has_builtin) && __has_builtin(__builtin_available) # define HAVE_FSTATAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) @@ -3008,7 +3008,7 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, } if (follow_symlinks_specified("access", follow_symlinks)) return -1; - + if (effective_ids) { argument_unavailable_error("access", "effective_ids"); return -1; @@ -3250,12 +3250,12 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (path->fd != -1) result = fchmod(path->fd, mode); else -#endif +#endif /* HAVE_CHMOD */ #ifdef HAVE_LCHMOD if ((!follow_symlinks) && (dir_fd == DEFAULT_DIR_FD)) result = lchmod(path->narrow, mode); else -#endif +#endif /* HAVE_LCHMOD */ #ifdef HAVE_FCHMODAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { if (HAVE_FCHMODAT_RUNTIME) { @@ -3285,10 +3285,9 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, result = -1; } -#endif } else -#endif +#endif /* HAVE_FHCMODAT */ result = chmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3297,10 +3296,10 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (fchmodat_unsupported) { if (dir_fd != DEFAULT_DIR_FD) { argument_unavailable_error("chmod", "dir_fd"); - return NULL; - } + return NULL; + } } - + if (fchmodat_nofollow_unsupported) { if (dir_fd != DEFAULT_DIR_FD) dir_fd_and_follow_symlinks_invalid("chmod", @@ -3310,9 +3309,10 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, return NULL; } else -#endif +#endif /* HAVE_FCHMODAT */ return path_error(path); } +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } @@ -3633,7 +3633,7 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, if (HAVE_FCHOWNAT_RUNTIME) { result = fchownat(dir_fd, path->narrow, uid, gid, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - } else { + } else { fchownat_unsupported = 1; } } else @@ -3940,14 +3940,14 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, result = linkat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow, follow_symlinks ? AT_SYMLINK_FOLLOW : 0); - + } #ifdef __APPLE__ else { if (src_dir_fd == DEFAULT_DIR_FD && dst_dir_fd == DEFAULT_DIR_FD) { /* See issue 41355: This matches the behaviour of !HAVE_LINKAT */ result = link(src->narrow, dst->narrow); - } else { + } else { linkat_unavailable = 1; } } @@ -3964,7 +3964,7 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, if (src_dir_fd != DEFAULT_DIR_FD) { argument_unavailable_error("link", "src_dir_fd"); } else { - argument_unavailable_error("link", "dst_dir_fd"); + argument_unavailable_error("link", "dst_dir_fd"); } return NULL; } @@ -5136,12 +5136,12 @@ utime_fd(utime_t *ut, int fd) UTIME_TO_TIMESPEC; return futimens(fd, time); - } else + } else #ifndef HAVE_FUTIMES { /* Not sure if this can happen */ PyErr_SetString( - PyExc_RuntimeError, + PyExc_RuntimeError, "neither futimens nor futimes are supported" " on this system"); return -1; @@ -5175,12 +5175,12 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) if (HAVE_UTIMENSAT_RUNTIME) { UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); - } else + } else #ifndef HAVE_LUTIMES { /* Not sure if this can happen */ PyErr_SetString( - PyExc_RuntimeError, + PyExc_RuntimeError, "neither utimensat nor lutimes are supported" " on this system"); return -1; @@ -8756,7 +8756,7 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, result = symlink(src->narrow, dst->narrow); Py_END_ALLOW_THREADS -#ifdef HAVE_SYMLINKAT +#ifdef HAVE_SYMLINKAT if (symlinkat_unavailable) { argument_unavailable_error(NULL, "dir_fd"); return NULL; @@ -13267,7 +13267,7 @@ _Py_COMP_DIAG_POP if (HAVE_FSTATAT_RUNTIME) { result = fstatat(self->dir_fd, path, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - } else + } else #endif /* HAVE_FSTATAT */ { @@ -15367,7 +15367,7 @@ posixmodule_exec(PyObject *m) { _posixstate *state = get_posix_state(m); -#if defined(HAVE_PWRITEV) +#if defined(HAVE_PWRITEV) if (HAVE_PWRITEV_RUNTIME) {} else { PyObject* dct = PyModule_GetDict(m); From 7e64e95bb2ca48999ca3612e9d3e2f3040c3517b Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sat, 31 Oct 2020 11:12:59 +0100 Subject: [PATCH 43/45] the sid attribute for posix_spawn is only available on recent versions of macOS --- Modules/posixmodule.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4b1724ece63778..4f11efb7b6e78d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -74,6 +74,8 @@ # define HAVE_UTIMENSAT_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) # define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +# define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) + #else /* Xcode 8 or earlier */ /* __builtin_available is not present in these compilers, but @@ -5842,6 +5844,9 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg } if (setsid) { +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + if (HAVE_POSIX_SPAWN_SETSID_RUNTIME) { +#endif #ifdef POSIX_SPAWN_SETSID all_flags |= POSIX_SPAWN_SETSID; #elif defined(POSIX_SPAWN_SETSID_NP) @@ -5850,6 +5855,14 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg argument_unavailable_error(func_name, "setsid"); return -1; #endif + +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + } else { + argument_unavailable_error(func_name, "setsid"); + return -1; + } +#endif /* HAVE_POSIX_SPAWN_SETSID_RUNTIME */ + } if (setsigmask) { From fef2e93a846b12cf01d625c5c32d07e33b03e907 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sat, 31 Oct 2020 11:44:43 +0100 Subject: [PATCH 44/45] Changes due to review by Victor Stinner --- Lib/test/test_posix.py | 11 +++-------- Lib/test/test_time.py | 19 ++++++++++--------- Mac/README.rst | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 117d67775f88e3..66e43f32edc7fc 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1909,7 +1909,9 @@ def test_posix_spawnp(self): class TestPosixWeaklinking(unittest.TestCase): # These test cases verify that weak linking support on macOS works # as expected. These cases only test new behaviour introduced by weak linking, - # regular behaviour is tested by the normal test cases. + # regular behaviour is tested by the normal test cases. + # + # See the section on Weak Linking in Mac/README.txt for more information. def setUp(self): import sysconfig import platform @@ -2027,9 +2029,6 @@ def test_link(self): finally: support.rmtree(base_path) - - - def test_listdir_scandir(self): self._verify_available("HAVE_FDOPENDIR") if self.mac_ver >= (10, 10): @@ -2139,10 +2138,6 @@ def test_utime(self): with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): os.utime("path", follow_symlinks=False) - # XXX: Check if this test is correct, the implementation of os.utime - # is fairly complicated - - def test_main(): try: diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 303bdf90aea05b..af68617bf44938 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1046,24 +1046,25 @@ class TestTimeWeaklinking(unittest.TestCase): # These test cases verify that weak linking support on macOS works # as expected. These cases only test new behaviour introduced by weak linking, # regular behaviour is tested by the normal test cases. - def setUp(self): + # + # See the section on Weak Linking in Mac/README.txt for more information. + def test_clock_functions(self): import sysconfig import platform config_vars = sysconfig.get_config_vars() - self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] } - self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split(".")) + var_name = "HAVE_CLOCK_GETTIME" + if var_name not in config_vars or not config_vars[var_name]: + raise unittest.SkipTest(f"{var_name} is not available") - def _verify_available(self, name): - if name not in self.available: - raise unittest.SkipTest(f"{name} not weak-linked") + mac_ver = tuple(int(x) for x in platform.mac_ver()[0].split(".")) - def test_clock_functions(self): - self._verify_available("HAVE_CLOCK_GETTIME") clock_names = [ "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", "clock_settime_ns", "clock_getres"] - if self.mac_ver >= (10, 12): + + + if mac_ver >= (10, 12): for name in clock_names: self.assertTrue(hasattr(time, name), f"time.{name} is not available") diff --git a/Mac/README.rst b/Mac/README.rst index ec7d873df277d7..f3638aa0019aaf 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -120,6 +120,8 @@ support ppc (Xcode 4 on 10.6 and later systems). The flavor can be specified using the configure option ``--with-universal-archs=VALUE``. The following values are available: + * ``universal2``: ``arm64``, ``x86_64`` + * ``intel``: ``i386``, ``x86_64`` * ``intel-32``: ``i386`` @@ -155,6 +157,8 @@ following combinations of SDKs and universal-archs flavors are available: * 10.15 and later SDKs support ``intel-64`` only + * 11.0 and later SDKs support ``universal2`` + The makefile for a framework build will also install ``python3.x-32`` binaries when the universal architecture includes at least one 32-bit architecture (that is, for all flavors but ``64-bit`` and ``intel-64``). @@ -352,6 +356,39 @@ A framework install also installs some applications in ``/Applications/Python X. And lastly a framework installation installs files in ``/usr/local/bin``, all of them symbolic links to files in ``/Library/Frameworks/Python.framework/Versions/X.Y/bin``. +Weak linking support +==================== + +The CPython sources support building with the latest SDK while targetting deployment +to macOS 10.9. This is done through weak linking of symbols introduced in macOS +10.10 or later and checking for their availability at runtime. + +This requires the use of Apple's compiler toolchain on macOS 10.13 or later. + +The basic implementation pattern is: + +* ``HAVE_`` is a macro defined (or not) by the configure script + +* ``HAVE__RUNTIME`` is a macro defined in the relevant source + files. This expands to a call to ``__builtin_available`` when using + a new enough Apple compiler, and to a true value otherwise. + +* Use ``HAVE__RUNTIME`` before calling ````. This macro + *must* be used a the sole expression in an if statement:: + + if (HAVE__RUNTIME) { + /* is available */ + } + + Or: + + if (HAVE__RUNTIME) {} else { + /* is not available */ + } + + Using other patterns (such as ``!HAVE__RUNTIME``) is not supported + by Apple's compilers. + Resources ========= From 296666b90de3b7eff3bc1e2fd4b29a048f7b7ea7 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 1 Nov 2020 16:51:37 +0100 Subject: [PATCH 45/45] inor changes - Review comments by Serhiy Storchaka - Test_posix utime tests were ignored by testing for the wrong feature macro. This uncovered a problem with the test case when running on macOS 10.9 - Added blurb --- Lib/_osx_support.py | 35 ++++++++++++------- Lib/test/test_posix.py | 27 ++++++-------- Lib/test/test_time.py | 1 - Mac/BuildScript/build-installer.py | 2 +- .../2020-11-01-16-40-23.bpo-41100.BApztP.rst | 8 +++++ Modules/_ctypes/callbacks.c | 2 +- Modules/_ctypes/callproc.c | 15 ++++---- Modules/posixmodule.c | 4 +-- 8 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 89b736224a9ceb..8a696ee9895e9b 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -110,6 +110,26 @@ def _get_system_version(): return _SYSTEM_VERSION +_SYSTEM_VERSION_TUPLE = None +def _get_system_version_tuple(): + """ + Return the macOS system version as a tuple + + The return value is safe to use to compare + two version numbers. + """ + global _SYSTEM_VERSION_TUPLE + if _SYSTEM_VERSION_TUPLE is None: + osx_version = _get_system_version() + if osx_version: + try: + _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + _SYSTEM_VERSION_TUPLE = () + + return _SYSTEM_VERSION_TUPLE + + def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. @@ -132,12 +152,7 @@ def _supports_universal_builds(): # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return bool(osx_version >= (10, 4)) if osx_version else False def _supports_arm64_builds(): @@ -146,12 +161,7 @@ def _supports_arm64_builds(): # 1. macOS 11 and later, unconditionally # 2. macOS 10.15 with Xcode 12.2 or later # For now the second category is ignored. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return osx_version >= (11, 0) if osx_version else False @@ -350,7 +360,6 @@ def compiler_fixup(compiler_so, cc_args): for idx in range(len(compiler_so)): if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": del compiler_so[idx:idx+2] - break if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 42ced6e677da34..a522717751ac17 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -2024,12 +2024,10 @@ def test_link(self): os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag - base_path = os.path.abspath(support.TESTFN) + '.link' - link_path = os.path.join(base_path, "link") - target_path = os.path.join(base_path, "target") - source_path = os.path.join(base_path, "source") - try: - os.mkdir(base_path) + with os_helper.temp_dir() as base_path: + link_path = os.path.join(base_path, "link") + target_path = os.path.join(base_path, "target") + source_path = os.path.join(base_path, "source") with open(source_path, "w") as fp: fp.write("data") @@ -2046,8 +2044,6 @@ def test_link(self): with self.assertRaises(FileExistsError): os.link(source_path, link_path, follow_symlinks=False) - finally: - support.rmtree(base_path) def test_listdir_scandir(self): self._verify_available("HAVE_FDOPENDIR") @@ -2142,22 +2138,19 @@ def test_symlink(self): os.symlink("a", "b", dir_fd=0) def test_utime(self): - self._verify_available("HAVE_FUTIMENS_RUNTIME") - self._verify_available("HAVE_UTIMENSAT_RUNTIME") + self._verify_available("HAVE_FUTIMENS") + self._verify_available("HAVE_UTIMENSAT") if self.mac_ver >= (10, 13): - self.assertIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) - self.assertIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + self.assertIn("HAVE_FUTIMENS", posix._have_functions) + self.assertIn("HAVE_UTIMENSAT", posix._have_functions) else: - self.assertNotIn("HAVE_FUTIMENS_RUNTIME", posix._have_functions) - self.assertNotIn("HAVE_UTIMENSAT_RUNTIME", posix._have_functions) + self.assertNotIn("HAVE_FUTIMENS", posix._have_functions) + self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions) with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.utime("path", dir_fd=0) - with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): - os.utime("path", follow_symlinks=False) - def test_main(): try: diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index af68617bf44938..325829864851c3 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1062,7 +1062,6 @@ def test_clock_functions(self): clock_names = [ "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", "clock_settime_ns", "clock_getres"] - if mac_ver >= (10, 12): for name in clock_names: diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 795769c0c796d8..0e76d3ca5bbc2e 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -828,7 +828,7 @@ def build_openssl_arch(archbase, arch): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], - "arm64": ["darwin64-arm64-cc" ], + "arm64": ["darwin64-arm64-cc"], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } diff --git a/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst new file mode 100644 index 00000000000000..6cbb279e7625ea --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst @@ -0,0 +1,8 @@ +Add support for macOS 11 and Apple Silicon systems. + +It is now possible to build "Universal 2" binaries using +"--enable-universalsdk --with-universal-archs=universal2". + +Binaries build on later macOS versions can be deployed back to older +versions (tested up to macOS 10.9), when using the correct deployment +target. This is tested using Xcode 11 and later. diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 77d4855277ecec..39cace58294d58 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -413,7 +413,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, # if USING_APPLE_OS_LIBFFI # define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) # else -# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME 1 # endif if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 09f97d682af7ed..a52d343031a090 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1467,6 +1467,8 @@ static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *ar char *name_str; if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { + int r; + if (!PyArg_ParseTuple(args, "O", &name)) return NULL; @@ -1475,15 +1477,16 @@ static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *ar if (PyUnicode_FSConverter(name, &name2) == 0) return NULL; - if (PyBytes_Check(name2)) - name_str = PyBytes_AS_STRING(name2); - else - name_str = PyByteArray_AS_STRING(name2); + name_str = PyBytes_AS_STRING(name2); - if(_dyld_shared_cache_contains_path(name_str)) + r = _dyld_shared_cache_contains_path(name_str); + Py_DECREF(name2); + + if (r) { Py_RETURN_TRUE; - else + } else { Py_RETURN_FALSE; + } } else { PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c1a12d8c54ac43..91ee3adea42f9f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2471,7 +2471,7 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, STRUCT_STAT st; int result; -#if defined(HAVE_FSTATAT) +#ifdef HAVE_FSTATAT int fstatat_unavailable = 0; #endif @@ -2990,7 +2990,7 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, result = access(path->narrow, mode); Py_END_ALLOW_THREADS -#if defined(HAVE_FACCESSAT) +#ifdef HAVE_FACCESSAT if (faccessat_unavailable) { if (dir_fd != DEFAULT_DIR_FD) { argument_unavailable_error("access", "dir_fd");