diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c9ab77269..c48691af1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -50,6 +50,14 @@ jobs: image: [linux, windows, macOs] pypy3: image: [linux, windows, macOs] + homebrew_py3: + image: [macOs] + py: brew@python3 + toxenv: py37 + homebrew_py2: + image: [macOs] + py: brew@python2 + toxenv: py27 fix_lint: image: [linux, windows] docs: diff --git a/docs/changelog/1561.bugfix.rst b/docs/changelog/1561.bugfix.rst new file mode 100644 index 000000000..c3c618545 --- /dev/null +++ b/docs/changelog/1561.bugfix.rst @@ -0,0 +1 @@ +Add macOs Python 2 Framework support (now we test it with the CI via brew) - by :user:`gaborbernat` diff --git a/docs/changelog/1641.feature.rst b/docs/changelog/1641.feature.rst new file mode 100644 index 000000000..f29c510a4 --- /dev/null +++ b/docs/changelog/1641.feature.rst @@ -0,0 +1 @@ +Report of the created virtual environment is now split across four short lines rather than one long - by :user:`gaborbernat` diff --git a/setup.cfg b/setup.cfg index b7f762315..1737e240d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,7 @@ virtualenv.create = cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows cpython2-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Posix + cpython2-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython2macOsFramework cpython2-win = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Windows pypy2-posix = virtualenv.create.via_global_ref.builtin.pypy.pypy2:PyPy2Posix pypy2-win = virtualenv.create.via_global_ref.builtin.pypy.pypy2:Pypy2Windows diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index d1c02ee07..267ddfc51 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -2,6 +2,7 @@ import argparse import logging +import os import sys from datetime import datetime @@ -17,12 +18,7 @@ def run(args=None, options=None): args = sys.argv[1:] try: session = cli_run(args, options) - logging.warning( - "created virtual environment in %.0fms %s with seeder %s", - (datetime.now() - start).total_seconds() * 1000, - ensure_text(str(session.creator)), - ensure_text(str(session.seeder)), - ) + logging.warning(LogSession(session, start)) except ProcessCallFailed as exception: print("subprocess call failed for {}".format(exception.cmd)) print(exception.out, file=sys.stdout, end="") @@ -30,6 +26,24 @@ def run(args=None, options=None): raise SystemExit(exception.code) +class LogSession(object): + def __init__(self, session, start): + self.session = session + self.start = start + + def __str__(self): + spec = self.session.creator.interpreter.spec + elapsed = (datetime.now() - self.start).total_seconds() * 1000 + lines = [ + "created virtual environment {} in {:.0f}ms".format(spec, elapsed), + " creator {}".format(ensure_text(str(self.session.creator))), + " seeder {}".format(ensure_text(str(self.session.seeder)),), + ] + if self.session.activators: + lines.append(" activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators))) + return os.linesep.join(lines) + + def run_with_catch(args=None): options = argparse.Namespace() try: diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py index a9bd52c6c..887263eb5 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py @@ -50,8 +50,17 @@ def ensure_directories(self): return dirs +def is_mac_os_framework(interpreter): + framework = bool(interpreter.sysconfig_vars["PYTHONFRAMEWORK"]) + return framework and interpreter.platform == "darwin" + + class CPython2Posix(CPython2, CPythonPosix): - """CPython 2 on POSIX""" + """CPython 2 on POSIX (excluding macOs framework builds)""" + + @classmethod + def can_describe(cls, interpreter): + return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter) @classmethod def sources(cls, interpreter): diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py new file mode 100644 index 000000000..ff2895809 --- /dev/null +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +"""The Apple Framework builds require their own customization""" +import logging +import os +import struct +import subprocess + +from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest +from virtualenv.util.path import Path +from virtualenv.util.six import ensure_text + +from .cpython2 import CPython2, is_mac_os_framework + + +class CPython2macOsFramework(CPython2, CPythonPosix): + @classmethod + def can_describe(cls, interpreter): + return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter) + + def create(self): + super(CPython2macOsFramework, self).create() + + # change the install_name of the copied python executable + current = os.path.join(self.interpreter.prefix, "Python") + fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size) + + @classmethod + def sources(cls, interpreter): + for src in super(CPython2macOsFramework, cls).sources(interpreter): + yield src + + # landmark for exec_prefix + name = "lib-dynload" + yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib) + + # this must symlink to the host prefix Python + marker = Path(interpreter.prefix) / "Python" + ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True) + yield ref + + @classmethod + def _executables(cls, interpreter): + for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter): + # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the + # stub executable in ${sys.prefix}/bin. + # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951 + fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python" + yield fixed_host_exe, targets + + +def fix_mach_o(exe, current, new, max_size): + """ + https://en.wikipedia.org/wiki/Mach-O + + Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries, + dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and + faster access to information in the symbol table. + + Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more + segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle + references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an + 'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name. + + The basic structure—a list of variable-length "load commands" that reference pages of data elsewhere in the file—was + also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice + Lisp. + + With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes + binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac + OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions, + does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables + (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and + unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format. + """ + try: + logging.debug(u"change Mach-O for %s from %s to %s", ensure_text(exe), current, ensure_text(new)) + _builtin_change_mach_o(max_size)(exe, current, new) + except Exception as e: + logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e) + try: + cmd = ["install_name_tool", "-change", current, new, exe] + subprocess.check_call(cmd) + except Exception: + logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed") + raise + + +def _builtin_change_mach_o(maxint): + MH_MAGIC = 0xFEEDFACE + MH_CIGAM = 0xCEFAEDFE + MH_MAGIC_64 = 0xFEEDFACF + MH_CIGAM_64 = 0xCFFAEDFE + FAT_MAGIC = 0xCAFEBABE + BIG_ENDIAN = ">" + LITTLE_ENDIAN = "<" + LC_LOAD_DYLIB = 0xC + + class FileView(object): + """A proxy for file-like objects that exposes a given view of a file. Modified from macholib.""" + + def __init__(self, file_obj, start=0, size=maxint): + if isinstance(file_obj, FileView): + self._file_obj = file_obj._file_obj + else: + self._file_obj = file_obj + self._start = start + self._end = start + size + self._pos = 0 + + def __repr__(self): + return "".format(self._start, self._end, self._file_obj) + + def tell(self): + return self._pos + + def _checkwindow(self, seek_to, op): + if not (self._start <= seek_to <= self._end): + msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end) + raise IOError(msg) + + def seek(self, offset, whence=0): + seek_to = offset + if whence == os.SEEK_SET: + seek_to += self._start + elif whence == os.SEEK_CUR: + seek_to += self._start + self._pos + elif whence == os.SEEK_END: + seek_to += self._end + else: + raise IOError("Invalid whence argument to seek: {!r}".format(whence)) + self._checkwindow(seek_to, "seek") + self._file_obj.seek(seek_to) + self._pos = seek_to - self._start + + def write(self, content): + here = self._start + self._pos + self._checkwindow(here, "write") + self._checkwindow(here + len(content), "write") + self._file_obj.seek(here, os.SEEK_SET) + self._file_obj.write(content) + self._pos += len(content) + + def read(self, size=maxint): + assert size >= 0 + here = self._start + self._pos + self._checkwindow(here, "read") + size = min(size, self._end - here) + self._file_obj.seek(here, os.SEEK_SET) + read_bytes = self._file_obj.read(size) + self._pos += len(read_bytes) + return read_bytes + + def read_data(file, endian, num=1): + """Read a given number of 32-bits unsigned integers from the given file with the given endianness.""" + res = struct.unpack(endian + "L" * num, file.read(num * 4)) + if len(res) == 1: + return res[0] + return res + + def mach_o_change(at_path, what, value): + """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value), + provided it's shorter.""" + + def do_macho(file, bits, endian): + # Read Mach-O header (the magic number is assumed read by the caller) + cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6) + # 64-bits header has one more field. + if bits == 64: + read_data(file, endian) + # The header is followed by n commands + for _ in range(n_commands): + where = file.tell() + # Read command header + cmd, cmd_size = read_data(file, endian, 2) + if cmd == LC_LOAD_DYLIB: + # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the + # beginning of the command. + name_offset = read_data(file, endian) + file.seek(where + name_offset, os.SEEK_SET) + # Read the NUL terminated string + load = file.read(cmd_size - name_offset).decode() + load = load[: load.index("\0")] + # If the string is what is being replaced, overwrite it. + if load == what: + file.seek(where + name_offset, os.SEEK_SET) + file.write(value.encode() + b"\0") + # Seek to the next command + file.seek(where + cmd_size, os.SEEK_SET) + + def do_file(file, offset=0, size=maxint): + file = FileView(file, offset, size) + # Read magic number + magic = read_data(file, BIG_ENDIAN) + if magic == FAT_MAGIC: + # Fat binaries contain nfat_arch Mach-O binaries + n_fat_arch = read_data(file, BIG_ENDIAN) + for _ in range(n_fat_arch): + # Read arch header + cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5) + do_file(file, offset, size) + elif magic == MH_MAGIC: + do_macho(file, 32, BIG_ENDIAN) + elif magic == MH_CIGAM: + do_macho(file, 32, LITTLE_ENDIAN) + elif magic == MH_MAGIC_64: + do_macho(file, 64, BIG_ENDIAN) + elif magic == MH_CIGAM_64: + do_macho(file, 64, LITTLE_ENDIAN) + + assert len(what) >= len(value) + + with open(at_path, "r+b") as f: + do_file(f) + + return mach_o_change diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py index e917ca58c..1f57aef67 100644 --- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py +++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py @@ -32,7 +32,8 @@ def create(self): else: custom_site_text = custom_site.read_text() expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs]) - site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)) + custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected) + site_py.write_text(custom_site_text) @classmethod def sources(cls, interpreter): diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/site.py b/src/virtualenv/create/via_global_ref/builtin/python2/site.py index 6ec015226..44654d26e 100644 --- a/src/virtualenv/create/via_global_ref/builtin/python2/site.py +++ b/src/virtualenv/create/via_global_ref/builtin/python2/site.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ A simple shim module to fix up things on Python 2 only. @@ -18,6 +19,7 @@ def main(): load_host_site() if global_site_package_enabled: add_global_site_package() + fix_install() def load_host_site(): @@ -37,8 +39,7 @@ def load_host_site(): here = __file__ # the distutils.install patterns will be injected relative to this site.py, save it here - with PatchForAppleFrameworkBuilds(): - reload(sys.modules["site"]) # noqa + reload(sys.modules["site"]) # noqa # call system site.py to setup import libraries # and then if the distutils site packages are not on the sys.path we add them via add_site_dir; note we must add # them by invoking add_site_dir to trigger the processing of pth files @@ -56,28 +57,12 @@ def load_host_site(): add_site_dir(full_path) -class PatchForAppleFrameworkBuilds(object): - """Apple Framework builds unconditionally add the global site-package, escape this behaviour""" - - framework = None - - def __enter__(self): - if sys.platform == "darwin": - from sysconfig import get_config_var - - self.framework = get_config_var("PYTHONFRAMEWORK") - if self.framework: - sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = None - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.framework: - sys.modules["sysconfig"]._CONFIG_VARS["PYTHONFRAMEWORK"] = self.framework +sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version def read_pyvenv(): """read pyvenv.cfg""" - os_sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version - config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep) + config_file = "{}{}pyvenv.cfg".format(sys.prefix, sep) with open(config_file) as file_handler: lines = file_handler.readlines() config = {} @@ -93,23 +78,41 @@ def read_pyvenv(): def rewrite_standard_library_sys_path(): """Once this site file is loaded the standard library paths have already been set, fix them up""" - sep = "\\" if sys.platform == "win32" else "/" - exe_dir = sys.executable[: sys.executable.rfind(sep)] + exe = abs_path(sys.executable) + exe_dir = exe[: exe.rfind(sep)] + prefix, exec_prefix = abs_path(sys.prefix), abs_path(sys.exec_prefix) + base_prefix, base_exec_prefix = abs_path(sys.base_prefix), abs_path(sys.base_exec_prefix) + base_executable = abs_path(sys.base_executable) for at, value in enumerate(sys.path): + value = abs_path(value) # replace old sys prefix path starts with new if value == exe_dir: pass # don't fix the current executable location, notably on Windows this gets added elif value.startswith(exe_dir): # content inside the exe folder needs to remap to original executables folder - orig_exe_folder = sys.base_executable[: sys.base_executable.rfind(sep)] + orig_exe_folder = base_executable[: base_executable.rfind(sep)] value = "{}{}".format(orig_exe_folder, value[len(exe_dir) :]) - elif value.startswith(sys.prefix): - value = "{}{}".format(sys.base_prefix, value[len(sys.prefix) :]) - elif value.startswith(sys.exec_prefix): - value = "{}{}".format(sys.base_exec_prefix, value[len(sys.exec_prefix) :]) + elif value.startswith(prefix): + value = "{}{}".format(base_prefix, value[len(prefix) :]) + elif value.startswith(exec_prefix): + value = "{}{}".format(base_exec_prefix, value[len(exec_prefix) :]) sys.path[at] = value +def abs_path(value): + keep = [] + values = value.split(sep) + i = len(values) - 1 + while i >= 0: + if values[i] == "..": + i -= 1 + else: + keep.append(values[i]) + i -= 1 + value = sep.join(keep[::-1]) + return value + + def disable_user_site_package(): """Flip the switch on enable user site package""" # sys.flags is a c-extension type, so we cannot monkeypatch it, replace it with a python class to flip it @@ -140,4 +143,29 @@ def add_global_site_package(): site.PREFIXES = orig_prefixes +def fix_install(): + def patch(dist_of): + # we cannot allow the prefix override as that would get packages installed outside of the virtual environment + old_parse_config_files = dist_of.Distribution.parse_config_files + + def parse_config_files(self, *args, **kwargs): + result = old_parse_config_files(self, *args, **kwargs) + install_dict = self.get_option_dict("install") + if "prefix" in install_dict: + install_dict["prefix"] = "virtualenv.patch", abs_path(sys.prefix) + return result + + dist_of.Distribution.parse_config_files = parse_config_files + + from distutils import dist + + patch(dist) + try: + from setuptools import dist + + patch(dist) + except ImportError: + pass # if setuptools is not around that's alright, just don't patch + + main() diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py index a579c4b37..15a644aa7 100644 --- a/src/virtualenv/create/via_global_ref/builtin/ref.py +++ b/src/virtualenv/create/via_global_ref/builtin/ref.py @@ -1,3 +1,8 @@ +""" +Virtual environments in the traditional sense are built as reference to the host python. This file allows declarative +references to elements on the file system, allowing our system to automatically detect what modes it can support given +the constraints: e.g. can the file system symlink, can the files be read, executed, etc. +""" from __future__ import absolute_import, unicode_literals import os @@ -14,15 +19,21 @@ @add_metaclass(ABCMeta) class PathRef(object): + """Base class that checks if a file reference can be symlink/copied""" + FS_SUPPORTS_SYMLINK = fs_supports_symlink() FS_CASE_SENSITIVE = fs_is_case_sensitive() - def __init__(self, src): + def __init__(self, src, must_symlink, must_copy): + self.must_symlink = must_symlink + self.must_copy = must_copy self.src = src self.exists = src.exists() self._can_read = None if self.exists else False self._can_copy = None if self.exists else False self._can_symlink = None if self.exists else False + if self.must_copy is True and self.must_symlink is True: + raise ValueError("can copy and symlink at the same time") def __repr__(self): return "{}(src={})".format(self.__class__.__name__, self.src) @@ -43,24 +54,39 @@ def can_read(self): @property def can_copy(self): if self._can_copy is None: - self._can_copy = self.can_read + if self.must_symlink: + self._can_copy = self.can_symlink + else: + self._can_copy = self.can_read return self._can_copy @property def can_symlink(self): if self._can_symlink is None: - self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read + if self.must_copy: + self._can_symlink = self.can_copy + else: + self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read return self._can_symlink @abstractmethod def run(self, creator, symlinks): raise NotImplementedError + def method(self, symlinks): + if self.must_symlink: + return symlink + if self.must_copy: + return copy + return symlink if symlinks else copy + @add_metaclass(ABCMeta) class ExePathRef(PathRef): - def __init__(self, src): - super(ExePathRef, self).__init__(src) + """Base class that checks if a executable can be references via symlink/copy""" + + def __init__(self, src, must_symlink, must_copy): + super(ExePathRef, self).__init__(src, must_symlink, must_copy) self._can_run = None @property @@ -83,22 +109,26 @@ def can_run(self): class PathRefToDest(PathRef): - def __init__(self, src, dest): - super(PathRefToDest, self).__init__(src) + """Link a path on the file system""" + + def __init__(self, src, dest, must_symlink=False, must_copy=False): + super(PathRefToDest, self).__init__(src, must_symlink, must_copy) self.dest = dest def run(self, creator, symlinks): dest = self.dest(creator, self.src) - method = symlink if symlinks else copy + method = self.method(symlinks) dest_iterable = dest if isinstance(dest, list) else (dest,) for dst in dest_iterable: method(self.src, dst) class ExePathRefToDest(PathRefToDest, ExePathRef): - def __init__(self, src, targets, dest, must_copy=False): - ExePathRef.__init__(self, src) - PathRefToDest.__init__(self, src, dest) + """Link a exe path on the file system""" + + def __init__(self, src, targets, dest, must_symlink=False, must_copy=False): + ExePathRef.__init__(self, src, must_symlink, must_copy) + PathRefToDest.__init__(self, src, dest, must_symlink, must_copy) if not self.FS_CASE_SENSITIVE: targets = list(OrderedDict((i.lower(), None) for i in targets).keys()) self.base = targets[0] @@ -108,8 +138,8 @@ def __init__(self, src, targets, dest, must_copy=False): def run(self, creator, symlinks): bin_dir = self.dest(creator, self.src).parent - method = symlink if self.must_copy is False and symlinks else copy dest = bin_dir / self.base + method = self.method(symlinks) method(self.src, dest) make_exe(dest) for extra in self.aliases: diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index c897cf178..3ffe57e6b 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -79,8 +79,9 @@ def abs_path(v): for element in self.sysconfig_paths.values(): for k in _CONF_VAR_RE.findall(element): config_var_keys.add(u(k[1:-1])) + config_var_keys.add("PYTHONFRAMEWORK") - self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i)) for i in config_var_keys} + self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i) or "") for i in config_var_keys} if self.implementation == "PyPy" and sys.version_info.major == 2: self.sysconfig_vars[u"implementation_lower"] = u"python" @@ -89,6 +90,7 @@ def abs_path(v): "stdlib", {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}, ) + self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) self._creators = None def _fast_get_system_executable(self): @@ -187,12 +189,7 @@ def __str__(self): ", ".join( "{}={}".format(k, v) for k, v in ( - ( - "spec", - "{}{}-{}".format( - self.implementation, ".".join(str(i) for i in self.version_info), self.architecture - ), - ), + ("spec", self.spec,), ( "system" if self.system_executable is not None and self.system_executable != self.executable @@ -218,6 +215,10 @@ def __str__(self): ) return content + @property + def spec(self): + return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) + @classmethod def clear_cache(cls): # this method is not used by itself, so here and called functions can import stuff locally diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 708b56a4f..753eeb25b 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -80,13 +80,15 @@ def add_parser_arguments(cls, parser, interpreter): def __unicode__(self): result = self.__class__.__name__ + result += "(" if self.extra_search_dir: - result += " extra search dirs = {}".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir)) + result += "extra_search_dir={},".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir)) + result += "download={},".format(self.download) for package in self.packages: - result += " {}{}".format( + result += " {}{},".format( package, "={}".format(getattr(self, "{}_version".format(package), None) or "latest") ) - return result + return result[:-1] + ")" def __repr__(self): return ensure_str(self.__unicode__()) diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index e8b80d7b8..8493c2004 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -105,6 +105,9 @@ def installer_class(self, pip_version): return CopyPipInstall def __unicode__(self): - return super(FromAppData, self).__unicode__() + " app_data_dir={} via={}".format( - self.app_data_dir.path, "symlink" if self.symlinks else "copy" + base = super(FromAppData, self).__unicode__() + return ( + base[:-1] + + ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data_dir.path) + + base[-1] ) diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index 76bd28278..ac7c49b24 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -68,7 +68,7 @@ def test_destination_not_write_able(tmp_path, capsys): def cleanup_sys_path(paths): from virtualenv.create.creator import HERE - paths = [Path(os.path.abspath(i)) for i in paths] + paths = [p.resolve() for p in (Path(os.path.abspath(i)) for i in paths) if p.exists()] to_remove = [Path(HERE)] if os.environ.get(str("PYCHARM_HELPERS_DIR")): to_remove.append(Path(os.environ[str("PYCHARM_HELPERS_DIR")]).parent) @@ -152,11 +152,12 @@ def test_create_no_seed(python, creator, isolated, system, coverage_env, special assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr # ensure the global site package is added or not, depending on flag - last_from_system_path = next(j for j in reversed(system_sys_path) if str(j).startswith(system["sys"]["prefix"])) + global_sys_path = system_sys_path[-1] if isolated == "isolated": - assert last_from_system_path not in sys_path, "last from system sys path {} is in venv sys path:\n{}".format( - ensure_text(str(last_from_system_path)), "\n".join(ensure_text(str(j)) for j in sys_path) + msg = "global sys path {} is in virtual environment sys path:\n{}".format( + ensure_text(str(global_sys_path)), "\n".join(ensure_text(str(j)) for j in sys_path) ) + assert global_sys_path not in sys_path, msg else: common = [] for left, right in zip(reversed(system_sys_path), reversed(sys_path)): diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 23ffb9af1..26d2e6932 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -131,10 +131,10 @@ def test_py_info_cached_symlink_error(mocker, tmp_path): def test_py_info_cache_clear(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") assert PythonInfo.from_exe(sys.executable) is not None - assert spy.call_count == 2 # at least two, one for the venv, one more for the host + assert spy.call_count >= 2 # at least two, one for the venv, one more for the host PythonInfo.clear_cache() assert PythonInfo.from_exe(sys.executable) is not None - assert spy.call_count == 4 + assert spy.call_count >= 4 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") @@ -142,14 +142,15 @@ def test_py_info_cached_symlink(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") first_result = PythonInfo.from_exe(sys.executable) assert first_result is not None - assert spy.call_count == 2 # at least two, one for the venv, one more for the host + count = spy.call_count + assert count >= 2 # at least two, one for the venv, one more for the host new_exe = tmp_path / "a" new_exe.symlink_to(sys.executable) new_exe_str = str(new_exe) second_result = PythonInfo.from_exe(new_exe_str) assert second_result.executable == new_exe_str - assert spy.call_count == 3 # no longer needed the host invocation, but the new symlink is must + assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must PyInfoMock = namedtuple("PyInfoMock", ["implementation", "architecture", "version_info"])