From 912f823ded14a9d5094852edfdb7c9ba56e1036a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 31 Jan 2022 15:21:00 +0100 Subject: [PATCH 01/20] bpo-45413: Define a "venv" sysconfig installation scheme The scheme is used for bootstrapping new virtual environments. It is a copy of the "posix_prefix" or "nt" install scheme depending on the platform. The venv module now uses that scheme to create new virtual environments instead of hardcoding the paths depending only on the platform. Downstream Python distributors customizing the "posix_prefix" or "nt" install scheme in a way that is not compatible with the install scheme used in virtual environments are encouraged to supply a modified working "venv" scheme. --- Lib/sysconfig.py | 22 +++++++++++++++++++ Lib/test/test_sysconfig.py | 2 +- Lib/venv/__init__.py | 17 ++++++-------- .../2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 9 ++++++++ 4 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index ef335c69421f5e..d047a460982857 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -58,6 +58,28 @@ }, } +# Downstream distributors can overwrite the default install scheme. +# This is done to support downstream modifications where distributors change +# the installation layout (eg. different site-packages directory). +# So, distributors will change the default scheme to one that correctly +# represents their layout. +# This presents an issue for projects/people that need to bootstrap virtual +# environments, like virtualenv. As distributors might now be customizing +# the default install scheme, there is no guarantee that the information +# returned by sysconfig.get_default_scheme/get_paths is correct for +# a virtual environment, the only guarantee we have is that it correct +# for the *current* environment. When bootstrapping a virtual environment, +# we need to know its layout, so that we can place the files in the +# correct locations. +# The "venv" install scheme is a scheme to bootstrap virtual environments, +# essentially identical to the default posix_prefix/nt schemes. +# Downstream distributors will need to change this, +# if their posix_prefix/nt scheme is not venv-compatible. +if os.name == 'nt': + _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['nt']) +else: + _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['posix_prefix']) + # NOTE: site.py has copy of this function. # Sync it when modify this function. diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2c4120979d9a27..3940aadb9c2aff 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -267,7 +267,7 @@ def test_get_config_h_filename(self): self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ['nt', 'posix_home', 'posix_prefix'] + wanted = ['nt', 'posix_home', 'posix_prefix', 'venv'] if HAS_USER_BASE: wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index b90765074c36d8..9e52cacbd3181a 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -16,6 +16,9 @@ CORE_VENV_DEPS = ('pip', 'setuptools') logger = logging.getLogger(__name__) +def _venv_path(name): + return sysconfig.get_path(name, scheme='venv').removeprefix(f'{sys.prefix}/') + class EnvBuilder: """ @@ -120,16 +123,10 @@ def create_if_needed(d): context.executable = executable context.python_dir = dirname context.python_exe = exename - if sys.platform == 'win32': - binname = 'Scripts' - incpath = 'Include' - libpath = os.path.join(env_dir, 'Lib', 'site-packages') - else: - binname = 'bin' - incpath = 'include' - libpath = os.path.join(env_dir, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') + binname = _venv_path('scripts') + incpath = _venv_path('include') + libpath = os.path.join(env_dir, _venv_path('purelib')) + context.inc_path = path = os.path.join(env_dir, incpath) create_if_needed(path) create_if_needed(libpath) diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst new file mode 100644 index 00000000000000..527e3765ca7481 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -0,0 +1,9 @@ +Define a ``"venv"`` :mod:`sysconfig` installation scheme to be used for +bootstrapping new virtual environments. It is a copy of the +``"posix_prefix"`` or ``"nt"`` install scheme depending on the platform. The +:mod:`venv` module now uses that scheme to create new virtual environments +instead of hardcoding the paths depending only on the platform. Downstream +Python distributors customizing the ``"posix_prefix"`` or ``"nt"`` install +scheme in a way that is not compatible with the install scheme used in +virtual environments are encouraged to supply a modified working ``"venv"`` +scheme. From 468689361eaf0d7f5200bbad9abe815384279f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 31 Jan 2022 15:54:15 +0100 Subject: [PATCH 02/20] Use the same conditional as previously in venv --- Lib/sysconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d047a460982857..b4daabb56f6067 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -75,7 +75,7 @@ # essentially identical to the default posix_prefix/nt schemes. # Downstream distributors will need to change this, # if their posix_prefix/nt scheme is not venv-compatible. -if os.name == 'nt': +if sys.platform == 'win32': _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['nt']) else: _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['posix_prefix']) From 6e98457ed74a26ce339772abccf535a72e2b6897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 31 Jan 2022 16:37:55 +0100 Subject: [PATCH 03/20] Use os.sep --- Lib/venv/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 9e52cacbd3181a..11130831a8e13d 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) def _venv_path(name): - return sysconfig.get_path(name, scheme='venv').removeprefix(f'{sys.prefix}/') + return sysconfig.get_path(name, scheme='venv').removeprefix(sys.prefix + os.sep) class EnvBuilder: From 602c2ad98428391c5460abda105d637040360605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 1 Feb 2022 19:28:38 +0100 Subject: [PATCH 04/20] A bit better wording of the comment --- Lib/sysconfig.py | 4 ++-- .../next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index b4daabb56f6067..4a4e346a412016 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -73,8 +73,8 @@ # correct locations. # The "venv" install scheme is a scheme to bootstrap virtual environments, # essentially identical to the default posix_prefix/nt schemes. -# Downstream distributors will need to change this, -# if their posix_prefix/nt scheme is not venv-compatible. +# Downstream distributors who patch posix_prefix/nt scheme are encouraged to do +# it after this copy is made: if sys.platform == 'win32': _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['nt']) else: diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index 527e3765ca7481..dcce785ebbbfb5 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -5,5 +5,5 @@ bootstrapping new virtual environments. It is a copy of the instead of hardcoding the paths depending only on the platform. Downstream Python distributors customizing the ``"posix_prefix"`` or ``"nt"`` install scheme in a way that is not compatible with the install scheme used in -virtual environments are encouraged to supply a modified working ``"venv"`` -scheme. +virtual environments are encouraged to do it after the ``"venv"`` scheme +is created or to supply a modified working ``"venv"`` scheme. From 44f588a2df67ccd965dc1b73951014096da3391c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 1 Feb 2022 20:04:15 +0100 Subject: [PATCH 05/20] Add a test that the venv scheme does not regress from the previously hardcoded values --- Lib/test/test_sysconfig.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 3940aadb9c2aff..b5bc65e9b1f24b 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -139,6 +139,37 @@ def test_get_preferred_schemes(self): self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) + def test_venv_scheme(self): + # The following directories were hardcoded in the venv module + # before bpo-45413, here we assert the venv scheme does not regress + if sys.platform == 'win32': + binname = 'Scripts' + incpath = 'Include' + libpath = os.path.join('Lib', 'site-packages') + else: + binname = 'bin' + incpath = 'include' + libpath = os.path.join('lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + + # Resolve the paths in prefix + binname = os.path.join(sys.prefix, binname) + incpath = os.path.join(sys.prefix, incpath) + libpath = os.path.join(sys.prefix, libpath) + + self.assertEqual(binname, sysconfig.get_path('scripts', scheme='venv')) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='venv')) + + # The include directory on POSIX isn't exactly the same as before, + # but it is "within" + sysconfig_includedir = sysconfig.get_path('include', scheme='venv') + if sys.platform == 'win32': + self.assertEqual(incpath, sysconfig.get_path('include', scheme='venv')) + else: + self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + + def test_get_config_vars(self): cvars = get_config_vars() self.assertIsInstance(cvars, dict) From 8af07ee2648127c3636fed39ba67b3f5b90677af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 1 Feb 2022 20:07:32 +0100 Subject: [PATCH 06/20] Revert "Use the same conditional as previously in venv" The tests have that one instead. This reverts commit 468689361eaf0d7f5200bbad9abe815384279f49. --- Lib/sysconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 4a4e346a412016..44fefa225aca5c 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -75,7 +75,7 @@ # essentially identical to the default posix_prefix/nt schemes. # Downstream distributors who patch posix_prefix/nt scheme are encouraged to do # it after this copy is made: -if sys.platform == 'win32': +if os.name == 'nt': _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['nt']) else: _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['posix_prefix']) From c73465830707ff3e95070a2aaa84b4afcc203434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 1 Feb 2022 23:54:52 +0100 Subject: [PATCH 07/20] venv: Resolve paths with env_dir as all bases --- Lib/venv/__init__.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 11130831a8e13d..a8640d9163fbed 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -16,9 +16,6 @@ CORE_VENV_DEPS = ('pip', 'setuptools') logger = logging.getLogger(__name__) -def _venv_path(name): - return sysconfig.get_path(name, scheme='venv').removeprefix(sys.prefix + os.sep) - class EnvBuilder: """ @@ -96,6 +93,15 @@ def clear_directory(self, path): elif os.path.isdir(fn): shutil.rmtree(fn) + def _venv_path(self, env_dir, name): + vars = { + 'base': env_dir, + 'platbase': env_dir, + 'installed_base': env_dir, + 'installed_platbase': env_dir, + } + return sysconfig.get_path(name, scheme='venv', vars=vars) + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -123,12 +129,12 @@ def create_if_needed(d): context.executable = executable context.python_dir = dirname context.python_exe = exename - binname = _venv_path('scripts') - incpath = _venv_path('include') - libpath = os.path.join(env_dir, _venv_path('purelib')) + binpath = self._venv_path(env_dir, 'scripts') + incpath = self._venv_path(env_dir, 'include') + libpath = self._venv_path(env_dir, 'purelib') - context.inc_path = path = os.path.join(env_dir, incpath) - create_if_needed(path) + context.inc_path = incpath + create_if_needed(incpath) create_if_needed(libpath) # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX if ((sys.maxsize > 2**32) and (os.name == 'posix') and @@ -136,8 +142,8 @@ def create_if_needed(d): link_path = os.path.join(env_dir, 'lib64') if not os.path.exists(link_path): # Issue #21643 os.symlink('lib', link_path) - context.bin_path = binpath = os.path.join(env_dir, binname) - context.bin_name = binname + context.bin_path = binpath + context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) # Assign and update the command to use when launching the newly created From 0a8556defef2793d878ec93cf188126bdf205a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 00:03:32 +0100 Subject: [PATCH 08/20] Optional: When we are in virtual environment, sysconfig.get_default_scheme() returns "venv" --- Lib/sysconfig.py | 2 ++ .../next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 44fefa225aca5c..7c5f0f994e3c8e 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -282,6 +282,8 @@ def get_preferred_scheme(key): def get_default_scheme(): + if sys.prefix != sys.base_prefix: + return 'venv' return get_preferred_scheme('prefix') diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index dcce785ebbbfb5..2c259ef9e9936a 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -7,3 +7,5 @@ Python distributors customizing the ``"posix_prefix"`` or ``"nt"`` install scheme in a way that is not compatible with the install scheme used in virtual environments are encouraged to do it after the ``"venv"`` scheme is created or to supply a modified working ``"venv"`` scheme. +When Python itself runs in a virtual environment, +:func:`sysconfig.get_default_scheme` returns ``"venv"``. From cb2ed5be96cc3b3d37e50c4e9e56cbdae1feb0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 01:05:29 +0100 Subject: [PATCH 09/20] Also return venv from get_preferred_scheme() --- Lib/sysconfig.py | 4 ++-- .../next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 7c5f0f994e3c8e..11e5e42f75e9f5 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -272,6 +272,8 @@ def _get_preferred_schemes(): def get_preferred_scheme(key): + if key == 'prefix' and sys.prefix != sys.base_prefix: + return 'venv' scheme = _get_preferred_schemes()[key] if scheme not in _INSTALL_SCHEMES: raise ValueError( @@ -282,8 +284,6 @@ def get_preferred_scheme(key): def get_default_scheme(): - if sys.prefix != sys.base_prefix: - return 'venv' return get_preferred_scheme('prefix') diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index 2c259ef9e9936a..e538072ce2fff3 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -8,4 +8,5 @@ scheme in a way that is not compatible with the install scheme used in virtual environments are encouraged to do it after the ``"venv"`` scheme is created or to supply a modified working ``"venv"`` scheme. When Python itself runs in a virtual environment, -:func:`sysconfig.get_default_scheme` returns ``"venv"``. +:func:`sysconfig.get_default_scheme` and +:func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns ``"venv"``. From af47953fed1e6a2e96ab8bb15e658b4975c9a8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 01:07:49 +0100 Subject: [PATCH 10/20] Rename binname to binpath in test for consistency with other names --- Lib/test/test_sysconfig.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b5bc65e9b1f24b..54b27bb6a476df 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -143,22 +143,22 @@ def test_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the venv scheme does not regress if sys.platform == 'win32': - binname = 'Scripts' + binpath = 'Scripts' incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') else: - binname = 'bin' + binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages') # Resolve the paths in prefix - binname = os.path.join(sys.prefix, binname) + binpath = os.path.join(sys.prefix, binpath) incpath = os.path.join(sys.prefix, incpath) libpath = os.path.join(sys.prefix, libpath) - self.assertEqual(binname, sysconfig.get_path('scripts', scheme='venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='venv')) self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='venv')) # The include directory on POSIX isn't exactly the same as before, From 4cc4015bf029ac3dcfa79230a944418d3d28c6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 11:44:14 +0100 Subject: [PATCH 11/20] Add documentation and what's new --- Doc/library/sysconfig.rst | 12 +++++++++++- Doc/library/venv.rst | 4 ++++ Doc/whatsnew/3.11.rst | 14 ++++++++++++++ .../2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 14 +++++++------- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 713be1e02cea6c..bc5863a7a1c786 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -73,7 +73,7 @@ Every new component that is installed using :mod:`distutils` or a Distutils-based system will follow the same scheme to copy its file in the right places. -Python currently supports six schemes: +Python currently supports seven schemes: - *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. @@ -86,6 +86,9 @@ Python currently supports six schemes: - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *osx_framework_user*: scheme for macOS, when the *user* option is used. +- *venv*: scheme for creating :mod:`Python virtual environments `; + by default, this is a copy of *posix_prefix* on POSIX + or a copy of *nt* for NT platforms. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: @@ -119,6 +122,9 @@ identifier. Python currently uses eight paths: This function was previously named ``_get_default_scheme()`` and considered an implementation detail. + .. versionchanged:: 3.11 + When Python runs form a virtual environment, + the *venv* scheme is returned. .. function:: get_preferred_scheme(key) @@ -132,6 +138,10 @@ identifier. Python currently uses eight paths: .. versionadded:: 3.10 + .. versionchanged:: 3.11 + When Python runs form a virtual environment and ``key="prefix"``, + the *venv* scheme is returned. + .. function:: _get_preferred_schemes() diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 092781b5ff1c45..de554d81aefb66 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -177,6 +177,10 @@ creation according to their needs, the :class:`EnvBuilder` class. ``clear=True``, contents of the environment directory will be cleared and then all necessary subdirectories will be recreated. + .. versionchanged:: 3.11 + The *venv* :ref:`sysconfig installation scheme ` + is used to construct the paths of the created directories. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index e054008753a948..8d8053d8073bc6 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -322,6 +322,20 @@ sys (equivalent to ``sys.exc_info()[1]``). (Contributed by Irit Katriel in :issue:`46328`.) + +sysconfig +--------- + +* A new *venv* :ref:`installation scheme ` was added + and is used when Python creates new virtual environments or when it + is running from a virtual environment. + This is useful for downstream distributors who modify + :func:`sysconfig.get_preferred_scheme`. + Third party code that creates new virtual environments should use this new + *venv* installation scheme to determine the paths, as does :mod:`venv`. + (Contributed by Miro Hrončok in :issue:`45413`.) + + threading --------- diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index e538072ce2fff3..b66a47ed178bf4 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -1,12 +1,12 @@ -Define a ``"venv"`` :mod:`sysconfig` installation scheme to be used for -bootstrapping new virtual environments. It is a copy of the -``"posix_prefix"`` or ``"nt"`` install scheme depending on the platform. The +Define a *venv* :ref:`sysconfig installation scheme ` +to be used for bootstrapping new virtual environments. It is a copy of the +*posix_prefix* or *nt* install scheme depending on the platform. The :mod:`venv` module now uses that scheme to create new virtual environments instead of hardcoding the paths depending only on the platform. Downstream -Python distributors customizing the ``"posix_prefix"`` or ``"nt"`` install +Python distributors customizing the *posix_prefix* or *nt* install scheme in a way that is not compatible with the install scheme used in -virtual environments are encouraged to do it after the ``"venv"`` scheme -is created or to supply a modified working ``"venv"`` scheme. +virtual environments are encouraged to do it after the *venv* scheme +is created or to supply a modified working *venv* scheme. When Python itself runs in a virtual environment, :func:`sysconfig.get_default_scheme` and -:func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns ``"venv"``. +:func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns *venv*. From 5d04c2267dcc01b9ad989895a1c69627cf89b330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 13:05:02 +0100 Subject: [PATCH 12/20] Also mention this change in What's new for venv --- Doc/whatsnew/3.11.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 8d8053d8073bc6..a3dea7a1845e11 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -370,6 +370,20 @@ unicodedata * The Unicode database has been updated to version 14.0.0. (:issue:`45190`). +venv +---- + +* When new Python virtual environments are created, + the *venv* :ref:`sysconfig installation scheme ` is used + to determine the paths inside the environment. + When Python runs in a virtual environment, the same installation scheme + is the default. + That means, downstream distributors can change the default sysconfig install + scheme without changing behavior of virtual environments. + Third party code that also creates new virtual environments should do the same. + (Contributed by Miro Hrončok in :issue:`45413`.) + + fcntl ----- From 5f35cd893791543a715cddd67304f33d088b2bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 13:15:01 +0100 Subject: [PATCH 13/20] Sysconfig docs: The venv scheme is also for running virtual environments --- Doc/library/sysconfig.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index bc5863a7a1c786..eac5c2270acf49 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -86,7 +86,7 @@ Python currently supports seven schemes: - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *osx_framework_user*: scheme for macOS, when the *user* option is used. -- *venv*: scheme for creating :mod:`Python virtual environments `; +- *venv*: scheme for :mod:`Python virtual environments `; by default, this is a copy of *posix_prefix* on POSIX or a copy of *nt* for NT platforms. From e7444a22b43b0d67d778b08bedb3f780f619b7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 13:16:22 +0100 Subject: [PATCH 14/20] Typo --- Doc/library/sysconfig.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index eac5c2270acf49..331a86381b8b09 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -88,7 +88,7 @@ Python currently supports seven schemes: - *osx_framework_user*: scheme for macOS, when the *user* option is used. - *venv*: scheme for :mod:`Python virtual environments `; by default, this is a copy of *posix_prefix* on POSIX - or a copy of *nt* for NT platforms. + or a copy of *nt* on NT platforms. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: From 30fdee3cb6812093ad6aa5821a3458aaff887b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Feb 2022 13:24:40 +0100 Subject: [PATCH 15/20] Assert the preferred(prefix) and default sysconfig install scheme in venv is *venv* --- Lib/test/test_venv.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 043158c79214b4..db812f21dbc562 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -236,6 +236,20 @@ def test_prefixes(self): out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode(), prefix) + @requireVenvCreate + def test_sysconfig_preferred_and_default_scheme(self): + """ + Test that the sysconfig preferred(prefix) and default scheme is venv. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + cmd = [envpy, '-c', None] + for call in ('get_preferred_scheme("prefix")', 'get_default_scheme()'): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd) + self.assertEqual(out.strip(), b'venv', err) + if sys.platform == 'win32': ENV_SUBDIRS = ( ('Scripts',), From 27ce189b521a2533a2ce404fc2fc0683fb0643ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 17 Feb 2022 13:57:09 +0100 Subject: [PATCH 16/20] Instead of *venv* scheme, have *posix_venv* and *nt_venv* --- Doc/library/sysconfig.rst | 23 +++++-- Doc/library/venv.rst | 3 +- Doc/whatsnew/3.11.rst | 15 ++-- Lib/sysconfig.py | 69 ++++++++++++------- Lib/test/test_sysconfig.py | 51 +++++++++----- Lib/test/test_venv.py | 8 ++- Lib/venv/__init__.py | 3 +- .../2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 17 +++-- 8 files changed, 124 insertions(+), 65 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 331a86381b8b09..1ac989b8509cab 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -73,7 +73,7 @@ Every new component that is installed using :mod:`distutils` or a Distutils-based system will follow the same scheme to copy its file in the right places. -Python currently supports seven schemes: +Python currently supports eight schemes: - *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. @@ -83,12 +83,13 @@ Python currently supports seven schemes: - *posix_user*: scheme for POSIX platforms used when a component is installed through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. +- *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX + platforms; by default it is the same as *posix_prefix* . - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. +- *nt_venv*: scheme for :mod:`Python virtual environments ` on NT + platforms; by default it is the same as *nt* . - *osx_framework_user*: scheme for macOS, when the *user* option is used. -- *venv*: scheme for :mod:`Python virtual environments `; - by default, this is a copy of *posix_prefix* on POSIX - or a copy of *nt* on NT platforms. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: @@ -124,7 +125,7 @@ identifier. Python currently uses eight paths: .. versionchanged:: 3.11 When Python runs form a virtual environment, - the *venv* scheme is returned. + the *posix_venv*/*nt_venv* scheme is returned. .. function:: get_preferred_scheme(key) @@ -140,7 +141,17 @@ identifier. Python currently uses eight paths: .. versionchanged:: 3.11 When Python runs form a virtual environment and ``key="prefix"``, - the *venv* scheme is returned. + the *posix_venv*/*nt_venv* scheme is returned. + +.. function:: get_venv_scheme() + + Return a preferred scheme name for an installation layout used inside + :mod:`Python virtual environments `. + + The return value is either ``"posix_venv"``, or ``"nt_venv"``, + depending on the platform. + + .. versionadded:: 3.11 .. function:: _get_preferred_schemes() diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index de554d81aefb66..ece91b1c237341 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -178,7 +178,8 @@ creation according to their needs, the :class:`EnvBuilder` class. and then all necessary subdirectories will be recreated. .. versionchanged:: 3.11 - The *venv* :ref:`sysconfig installation scheme ` + The *posix_venv*/*nt_venv* + :ref:`sysconfig installation scheme ` is used to construct the paths of the created directories. .. method:: create_configuration(context) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index a3dea7a1845e11..da3d4e31ce89e0 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -326,13 +326,16 @@ sys sysconfig --------- -* A new *venv* :ref:`installation scheme ` was added - and is used when Python creates new virtual environments or when it +* Two new :ref:`installation schemes ` + (*posix_venv* and *nt_venv*) were added and are used when Python creates + new virtual environments or when it is running from a virtual environment. + To determine what installation scheme is suitable for the platform Python + currently runs on, :func:`sysconfig.get_venv_scheme` was added. This is useful for downstream distributors who modify :func:`sysconfig.get_preferred_scheme`. - Third party code that creates new virtual environments should use this new - *venv* installation scheme to determine the paths, as does :mod:`venv`. + Third party code that creates new virtual environments should use the new + installation scheme to determine the paths, as does :mod:`venv`. (Contributed by Miro Hrončok in :issue:`45413`.) @@ -373,8 +376,8 @@ unicodedata venv ---- -* When new Python virtual environments are created, - the *venv* :ref:`sysconfig installation scheme ` is used +* When new Python virtual environments are created, the *posix_venv*/*nt_venv* + :ref:`sysconfig installation scheme ` is used to determine the paths inside the environment. When Python runs in a virtual environment, the same installation scheme is the default. diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 11e5e42f75e9f5..a365e437b02ed6 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -56,30 +56,47 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. + # This is done to support downstream modifications where distributors change + # the installation layout (eg. different site-packages directory). + # So, distributors will change the default scheme to one that correctly + # represents their layout. + # This presents an issue for projects/people that need to bootstrap virtual + # environments, like virtualenv. As distributors might now be customizing + # the default install scheme, there is no guarantee that the information + # returned by sysconfig.get_default_scheme/get_paths is correct for + # a virtual environment, the only guarantee we have is that it correct + # for the *current* environment. When bootstrapping a virtual environment, + # we need to know its layout, so that we can place the files in the + # correct locations. + # The "*_venv" install scheme is a scheme to bootstrap virtual environments, + # essentially identical to the default posix_prefix/nt schemes. + # Downstream distributors who patch posix_prefix/nt scheme are encouraged to + # Leave the following schemes unchanged + 'posix_venv': { + 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'include': + '{installed_base}/include/python{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/python{py_version_short}{abiflags}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'nt_venv': { + 'stdlib': '{installed_base}/Lib', + 'platstdlib': '{base}/Lib', + 'purelib': '{base}/Lib/site-packages', + 'platlib': '{base}/Lib/site-packages', + 'include': '{installed_base}/Include', + 'platinclude': '{installed_base}/Include', + 'scripts': '{base}/Scripts', + 'data': '{base}', + }, } -# Downstream distributors can overwrite the default install scheme. -# This is done to support downstream modifications where distributors change -# the installation layout (eg. different site-packages directory). -# So, distributors will change the default scheme to one that correctly -# represents their layout. -# This presents an issue for projects/people that need to bootstrap virtual -# environments, like virtualenv. As distributors might now be customizing -# the default install scheme, there is no guarantee that the information -# returned by sysconfig.get_default_scheme/get_paths is correct for -# a virtual environment, the only guarantee we have is that it correct -# for the *current* environment. When bootstrapping a virtual environment, -# we need to know its layout, so that we can place the files in the -# correct locations. -# The "venv" install scheme is a scheme to bootstrap virtual environments, -# essentially identical to the default posix_prefix/nt schemes. -# Downstream distributors who patch posix_prefix/nt scheme are encouraged to do -# it after this copy is made: -if os.name == 'nt': - _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['nt']) -else: - _INSTALL_SCHEMES['venv'] = dict(_INSTALL_SCHEMES['posix_prefix']) - # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -271,9 +288,15 @@ def _get_preferred_schemes(): } +def get_venv_scheme(): + if os.name == 'nt': + return 'nt_venv' + return 'posix_venv' + + def get_preferred_scheme(key): if key == 'prefix' and sys.prefix != sys.base_prefix: - return 'venv' + return get_venv_scheme() scheme = _get_preferred_schemes()[key] if scheme not in _INSTALL_SCHEMES: raise ValueError( diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 54b27bb6a476df..93bd6919e4ed16 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -139,36 +139,49 @@ def test_get_preferred_schemes(self): self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) - def test_venv_scheme(self): + def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module - # before bpo-45413, here we assert the venv scheme does not regress - if sys.platform == 'win32': - binpath = 'Scripts' - incpath = 'Include' - libpath = os.path.join('Lib', 'site-packages') - else: - binpath = 'bin' - incpath = 'include' - libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') + # before bpo-45413, here we assert the posix_venv scheme does not regress + binpath = 'bin' + incpath = 'include' + libpath = os.path.join('lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') # Resolve the paths in prefix binpath = os.path.join(sys.prefix, binpath) incpath = os.path.join(sys.prefix, incpath) libpath = os.path.join(sys.prefix, libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + + def test_nt_venv_scheme(self): + # The following directories were hardcoded in the venv module + # before bpo-45413, here we assert the posix_venv scheme does not regress + binpath = 'Scripts' + incpath = 'Include' + libpath = os.path.join('Lib', 'site-packages') + + # Resolve the paths in prefix + binpath = os.path.join(sys.prefix, binpath) + incpath = os.path.join(sys.prefix, incpath) + libpath = os.path.join(sys.prefix, libpath) + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + + def test_get_venv_scheme(self): if sys.platform == 'win32': - self.assertEqual(incpath, sysconfig.get_path('include', scheme='venv')) + self.assertEqual(sysconfig.get_venv_scheme(), 'nt_venv') else: - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) - + self.assertEqual(sysconfig.get_venv_scheme(), 'posix_venv') def test_get_config_vars(self): cvars = get_config_vars() @@ -298,7 +311,7 @@ def test_get_config_h_filename(self): self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ['nt', 'posix_home', 'posix_prefix', 'venv'] + wanted = ['nt', 'posix_home', 'posix_prefix', 'posix_venv', 'nt_venv'] if HAS_USER_BASE: wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index db812f21dbc562..9751183f3546c7 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -239,7 +239,8 @@ def test_prefixes(self): @requireVenvCreate def test_sysconfig_preferred_and_default_scheme(self): """ - Test that the sysconfig preferred(prefix) and default scheme is venv. + Test that the sysconfig preferred(prefix) and default scheme is + posix_venv/nt_venv. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) @@ -248,7 +249,10 @@ def test_sysconfig_preferred_and_default_scheme(self): for call in ('get_preferred_scheme("prefix")', 'get_default_scheme()'): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd) - self.assertEqual(out.strip(), b'venv', err) + if sys.platform == 'win32': + self.assertEqual(out.strip(), b'nt_venv', err) + else: + self.assertEqual(out.strip(), b'posix_venv', err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index a8640d9163fbed..2eb371160242f7 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -100,7 +100,8 @@ def _venv_path(self, env_dir, name): 'installed_base': env_dir, 'installed_platbase': env_dir, } - return sysconfig.get_path(name, scheme='venv', vars=vars) + venv_scheme = sysconfig.get_venv_scheme() + return sysconfig.get_path(name, scheme=venv_scheme, vars=vars) def ensure_directories(self, env_dir): """ diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index b66a47ed178bf4..38e6399f13d367 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -1,12 +1,15 @@ -Define a *venv* :ref:`sysconfig installation scheme ` -to be used for bootstrapping new virtual environments. It is a copy of the -*posix_prefix* or *nt* install scheme depending on the platform. The -:mod:`venv` module now uses that scheme to create new virtual environments +Define *posix_venv* and *nt_venv* +:ref:`sysconfig installation schemes ` +to be used for bootstrapping new virtual environments. +Add :func:`sysconfig.get_venv_scheme` function to get the appropriate one of them. +The schemes are identical to the pre-existing +*posix_prefix* and *nt* install schemes. +The :mod:`venv` module now uses the schemes to create new virtual environments instead of hardcoding the paths depending only on the platform. Downstream Python distributors customizing the *posix_prefix* or *nt* install scheme in a way that is not compatible with the install scheme used in -virtual environments are encouraged to do it after the *venv* scheme -is created or to supply a modified working *venv* scheme. +virtual environments are encouraged not to customize the *venv* schemes. When Python itself runs in a virtual environment, :func:`sysconfig.get_default_scheme` and -:func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns *venv*. +:func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns +*posix_venv* or *nt_venv*. From 62974493a7edf8f659c8e9f6e43d6dd832654aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 3 Mar 2022 14:30:02 +0100 Subject: [PATCH 17/20] Apply suggestions from language review Co-authored-by: Petr Viktorin --- Doc/library/sysconfig.rst | 4 ++-- Doc/whatsnew/3.11.rst | 2 +- Lib/sysconfig.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 1ac989b8509cab..75fb197f7dd622 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -124,7 +124,7 @@ identifier. Python currently uses eight paths: considered an implementation detail. .. versionchanged:: 3.11 - When Python runs form a virtual environment, + When Python runs from a virtual environment, the *posix_venv*/*nt_venv* scheme is returned. .. function:: get_preferred_scheme(key) @@ -140,7 +140,7 @@ identifier. Python currently uses eight paths: .. versionadded:: 3.10 .. versionchanged:: 3.11 - When Python runs form a virtual environment and ``key="prefix"``, + When Python runs from a virtual environment and ``key="prefix"``, the *posix_venv*/*nt_venv* scheme is returned. .. function:: get_venv_scheme() diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index da3d4e31ce89e0..c87e7dfad0b38d 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -381,7 +381,7 @@ venv to determine the paths inside the environment. When Python runs in a virtual environment, the same installation scheme is the default. - That means, downstream distributors can change the default sysconfig install + That means that downstream distributors can change the default sysconfig install scheme without changing behavior of virtual environments. Third party code that also creates new virtual environments should do the same. (Contributed by Miro Hrončok in :issue:`45413`.) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a365e437b02ed6..5041912f62ffaa 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -65,14 +65,14 @@ # environments, like virtualenv. As distributors might now be customizing # the default install scheme, there is no guarantee that the information # returned by sysconfig.get_default_scheme/get_paths is correct for - # a virtual environment, the only guarantee we have is that it correct + # a virtual environment, the only guarantee we have is that it is correct # for the *current* environment. When bootstrapping a virtual environment, # we need to know its layout, so that we can place the files in the # correct locations. # The "*_venv" install scheme is a scheme to bootstrap virtual environments, # essentially identical to the default posix_prefix/nt schemes. # Downstream distributors who patch posix_prefix/nt scheme are encouraged to - # Leave the following schemes unchanged + # leave the following schemes unchanged 'posix_venv': { 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', From 283db7a1f9ceef3d7e1bd9f601daaf5f84f3eefe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 17 Mar 2022 11:35:58 +0100 Subject: [PATCH 18/20] Drop get_venv_scheme() in favor of *venv* scheme --- Doc/library/sysconfig.rst | 14 ++------- Doc/whatsnew/3.11.rst | 13 ++++---- Lib/sysconfig.py | 14 ++++----- Lib/test/test_sysconfig.py | 30 ++++++++++++++++--- Lib/test/test_venv.py | 8 ++--- Lib/venv/__init__.py | 3 +- .../2022-01-31-15-19-38.bpo-45413.1vaS0V.rst | 6 ++-- 7 files changed, 49 insertions(+), 39 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 75fb197f7dd622..a0294e7809f6c5 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -73,7 +73,7 @@ Every new component that is installed using :mod:`distutils` or a Distutils-based system will follow the same scheme to copy its file in the right places. -Python currently supports eight schemes: +Python currently supports nine schemes: - *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. @@ -89,6 +89,8 @@ Python currently supports eight schemes: - *nt_user*: scheme for NT platforms, when the *user* option is used. - *nt_venv*: scheme for :mod:`Python virtual environments ` on NT platforms; by default it is the same as *nt* . +- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending + on the platform Python runs on - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -143,16 +145,6 @@ identifier. Python currently uses eight paths: When Python runs from a virtual environment and ``key="prefix"``, the *posix_venv*/*nt_venv* scheme is returned. -.. function:: get_venv_scheme() - - Return a preferred scheme name for an installation layout used inside - :mod:`Python virtual environments `. - - The return value is either ``"posix_venv"``, or ``"nt_venv"``, - depending on the platform. - - .. versionadded:: 3.11 - .. function:: _get_preferred_schemes() diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index c87e7dfad0b38d..ceef422befe3cc 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -327,15 +327,16 @@ sysconfig --------- * Two new :ref:`installation schemes ` - (*posix_venv* and *nt_venv*) were added and are used when Python creates - new virtual environments or when it - is running from a virtual environment. - To determine what installation scheme is suitable for the platform Python - currently runs on, :func:`sysconfig.get_venv_scheme` was added. + (*posix_venv*, *nt_venv* and *venv*) were added and are used when Python + creates new virtual environments or when it is running from a virtual + environment. + The first two schemes (*posix_venv* and *nt_venv*) are OS-specific + for non-Windoes and Windows, the *venv* is essentially an alias to one of + them according to the OS Python runs on. This is useful for downstream distributors who modify :func:`sysconfig.get_preferred_scheme`. Third party code that creates new virtual environments should use the new - installation scheme to determine the paths, as does :mod:`venv`. + *venv* installation scheme to determine the paths, as does :mod:`venv`. (Contributed by Miro Hrončok in :issue:`45413`.) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 5041912f62ffaa..105c2aaa8d1233 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -97,6 +97,12 @@ }, } +# For the OS-native venv scheme, we essentially provide an alias: +if os.name == 'nt': + _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] +else: + _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] + # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -288,15 +294,9 @@ def _get_preferred_schemes(): } -def get_venv_scheme(): - if os.name == 'nt': - return 'nt_venv' - return 'posix_venv' - - def get_preferred_scheme(key): if key == 'prefix' and sys.prefix != sys.base_prefix: - return get_venv_scheme() + return 'venv' scheme = _get_preferred_schemes()[key] if scheme not in _INSTALL_SCHEMES: raise ValueError( diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 93bd6919e4ed16..c7ec78fa4dc814 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -177,11 +177,33 @@ def test_nt_venv_scheme(self): self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) - def test_get_venv_scheme(self): + def test_venv_scheme(self): if sys.platform == 'win32': - self.assertEqual(sysconfig.get_venv_scheme(), 'nt_venv') + self.assertEqual( + sysconfig.get_path('scripts', scheme='venv'), + sysconfig.get_path('scripts', scheme='nt_venv') + ) + self.assertEqual( + sysconfig.get_path('include', scheme='venv'), + sysconfig.get_path('include', scheme='nt_venv') + ) + self.assertEqual( + sysconfig.get_path('purelib', scheme='venv'), + sysconfig.get_path('purelib', scheme='nt_venv') + ) else: - self.assertEqual(sysconfig.get_venv_scheme(), 'posix_venv') + self.assertEqual( + sysconfig.get_path('scripts', scheme='venv'), + sysconfig.get_path('scripts', scheme='posix_venv') + ) + self.assertEqual( + sysconfig.get_path('include', scheme='venv'), + sysconfig.get_path('include', scheme='posix_venv') + ) + self.assertEqual( + sysconfig.get_path('purelib', scheme='venv'), + sysconfig.get_path('purelib', scheme='posix_venv') + ) def test_get_config_vars(self): cvars = get_config_vars() @@ -311,7 +333,7 @@ def test_get_config_h_filename(self): self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ['nt', 'posix_home', 'posix_prefix', 'posix_venv', 'nt_venv'] + wanted = ['nt', 'posix_home', 'posix_prefix', 'posix_venv', 'nt_venv', 'venv'] if HAS_USER_BASE: wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9751183f3546c7..db812f21dbc562 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -239,8 +239,7 @@ def test_prefixes(self): @requireVenvCreate def test_sysconfig_preferred_and_default_scheme(self): """ - Test that the sysconfig preferred(prefix) and default scheme is - posix_venv/nt_venv. + Test that the sysconfig preferred(prefix) and default scheme is venv. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) @@ -249,10 +248,7 @@ def test_sysconfig_preferred_and_default_scheme(self): for call in ('get_preferred_scheme("prefix")', 'get_default_scheme()'): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd) - if sys.platform == 'win32': - self.assertEqual(out.strip(), b'nt_venv', err) - else: - self.assertEqual(out.strip(), b'posix_venv', err) + self.assertEqual(out.strip(), b'venv', err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 2eb371160242f7..a8640d9163fbed 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -100,8 +100,7 @@ def _venv_path(self, env_dir, name): 'installed_base': env_dir, 'installed_platbase': env_dir, } - venv_scheme = sysconfig.get_venv_scheme() - return sysconfig.get_path(name, scheme=venv_scheme, vars=vars) + return sysconfig.get_path(name, scheme='venv', vars=vars) def ensure_directories(self, env_dir): """ diff --git a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst index 38e6399f13d367..6daff85781a5d4 100644 --- a/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst +++ b/Misc/NEWS.d/next/Library/2022-01-31-15-19-38.bpo-45413.1vaS0V.rst @@ -1,10 +1,10 @@ Define *posix_venv* and *nt_venv* :ref:`sysconfig installation schemes ` to be used for bootstrapping new virtual environments. -Add :func:`sysconfig.get_venv_scheme` function to get the appropriate one of them. +Add *venv* sysconfig installation scheme to get the appropriate one of the above. The schemes are identical to the pre-existing *posix_prefix* and *nt* install schemes. -The :mod:`venv` module now uses the schemes to create new virtual environments +The :mod:`venv` module now uses the *venv* scheme to create new virtual environments instead of hardcoding the paths depending only on the platform. Downstream Python distributors customizing the *posix_prefix* or *nt* install scheme in a way that is not compatible with the install scheme used in @@ -12,4 +12,4 @@ virtual environments are encouraged not to customize the *venv* schemes. When Python itself runs in a virtual environment, :func:`sysconfig.get_default_scheme` and :func:`sysconfig.get_preferred_scheme` with ``key="prefix"`` returns -*posix_venv* or *nt_venv*. +*venv*. From 594c4d9882cf7e444917ddf8c3baa762a72a332d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 17 Mar 2022 11:57:46 +0100 Subject: [PATCH 19/20] Apply my own suggestions from docs review --- Doc/library/sysconfig.rst | 4 ++-- Doc/library/venv.rst | 2 +- Doc/whatsnew/3.11.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index a0294e7809f6c5..fa18d62d22af51 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -127,7 +127,7 @@ identifier. Python currently uses eight paths: .. versionchanged:: 3.11 When Python runs from a virtual environment, - the *posix_venv*/*nt_venv* scheme is returned. + the *venv* scheme is returned. .. function:: get_preferred_scheme(key) @@ -143,7 +143,7 @@ identifier. Python currently uses eight paths: .. versionchanged:: 3.11 When Python runs from a virtual environment and ``key="prefix"``, - the *posix_venv*/*nt_venv* scheme is returned. + the *venv* scheme is returned. .. function:: _get_preferred_schemes() diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index ece91b1c237341..b40bd4102c2593 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -178,7 +178,7 @@ creation according to their needs, the :class:`EnvBuilder` class. and then all necessary subdirectories will be recreated. .. versionchanged:: 3.11 - The *posix_venv*/*nt_venv* + The *venv* :ref:`sysconfig installation scheme ` is used to construct the paths of the created directories. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index ceef422befe3cc..f4580292d50e22 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -328,7 +328,7 @@ sysconfig * Two new :ref:`installation schemes ` (*posix_venv*, *nt_venv* and *venv*) were added and are used when Python - creates new virtual environments or when it is running from a virtual + creates new virtual environments or when it is running from a virtual environment. The first two schemes (*posix_venv* and *nt_venv*) are OS-specific for non-Windoes and Windows, the *venv* is essentially an alias to one of @@ -377,7 +377,7 @@ unicodedata venv ---- -* When new Python virtual environments are created, the *posix_venv*/*nt_venv* +* When new Python virtual environments are created, the *venv* :ref:`sysconfig installation scheme ` is used to determine the paths inside the environment. When Python runs in a virtual environment, the same installation scheme From 98909189b861656b1f6b9264e9674f5412f3109e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 17 Mar 2022 20:21:00 +0100 Subject: [PATCH 20/20] Typo Co-authored-by: Steve Dower --- Doc/whatsnew/3.11.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index f4580292d50e22..907b25ac3ac432 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -331,7 +331,7 @@ sysconfig creates new virtual environments or when it is running from a virtual environment. The first two schemes (*posix_venv* and *nt_venv*) are OS-specific - for non-Windoes and Windows, the *venv* is essentially an alias to one of + for non-Windows and Windows, the *venv* is essentially an alias to one of them according to the OS Python runs on. This is useful for downstream distributors who modify :func:`sysconfig.get_preferred_scheme`.