macOs Python 2 Framework support (#1641)

azure-pipelines.yml

@@ -50,6 +50,14 @@ jobs:
         image: [linux, windows, macOs]
       pypy3:
         image: [linux, windows, macOs]
+      homebrew_py3:
+        image: [macOs]
+        py: brew@python3
+        toxenv: py37
+      homebrew_py2:
+        image: [macOs]
+        py: brew@python2
+        toxenv: py27
       fix_lint:
         image: [linux, windows]
       docs:

docs/changelog/1561.bugfix.rst

@@ -0,0 +1 @@
+Add macOs Python 2 Framework support (now we test it with the CI via brew) - by :user:`gaborbernat`

docs/changelog/1641.feature.rst

@@ -0,0 +1 @@
+Report of the created virtual environment is now split across four short lines rather than one long - by :user:`gaborbernat`

setup.cfg

@@ -68,6 +68,7 @@ virtualenv.create =
     cpython3-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix
     cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows
     cpython2-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Posix
+    cpython2-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython2macOsFramework
     cpython2-win = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Windows
     pypy2-posix = virtualenv.create.via_global_ref.builtin.pypy.pypy2:PyPy2Posix
     pypy2-win = virtualenv.create.via_global_ref.builtin.pypy.pypy2:Pypy2Windows

src/virtualenv/__main__.py

@@ -2,6 +2,7 @@
 
 import argparse
 import logging
+import os
 import sys
 from datetime import datetime
 
@@ -17,19 +18,32 @@ def run(args=None, options=None):
         args = sys.argv[1:]
     try:
         session = cli_run(args, options)
-        logging.warning(
-            "created virtual environment in %.0fms %s with seeder %s",
-            (datetime.now() - start).total_seconds() * 1000,
-            ensure_text(str(session.creator)),
-            ensure_text(str(session.seeder)),
-        )
+        logging.warning(LogSession(session, start))
     except ProcessCallFailed as exception:
         print("subprocess call failed for {}".format(exception.cmd))
         print(exception.out, file=sys.stdout, end="")
         print(exception.err, file=sys.stderr, end="")
         raise SystemExit(exception.code)
 
 
+class LogSession(object):
+    def __init__(self, session, start):
+        self.session = session
+        self.start = start
+
+    def __str__(self):
+        spec = self.session.creator.interpreter.spec
+        elapsed = (datetime.now() - self.start).total_seconds() * 1000
+        lines = [
+            "created virtual environment {} in {:.0f}ms".format(spec, elapsed),
+            "  creator {}".format(ensure_text(str(self.session.creator))),
+            "  seeder {}".format(ensure_text(str(self.session.seeder)),),
+        ]
+        if self.session.activators:
+            lines.append("  activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators)))
+        return os.linesep.join(lines)
+
+
 def run_with_catch(args=None):
     options = argparse.Namespace()
     try:

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -50,8 +50,17 @@ def ensure_directories(self):
         return dirs
 
 
+def is_mac_os_framework(interpreter):
+    framework = bool(interpreter.sysconfig_vars["PYTHONFRAMEWORK"])
+    return framework and interpreter.platform == "darwin"
+
+
 class CPython2Posix(CPython2, CPythonPosix):
-    """CPython 2 on POSIX"""
+    """CPython 2 on POSIX (excluding macOs framework builds)"""
+
+    @classmethod
+    def can_describe(cls, interpreter):
+        return is_mac_os_framework(interpreter) is False and super(CPython2Posix, cls).can_describe(interpreter)
 
     @classmethod
     def sources(cls, interpreter):

src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+"""The Apple Framework builds require their own customization"""
+import logging
+import os
+import struct
+import subprocess
+
+from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.util.path import Path
+from virtualenv.util.six import ensure_text
+
+from .cpython2 import CPython2, is_mac_os_framework
+
+
+class CPython2macOsFramework(CPython2, CPythonPosix):
+    @classmethod
+    def can_describe(cls, interpreter):
+        return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter)
+
+    def create(self):
+        super(CPython2macOsFramework, self).create()
+
+        # change the install_name of the copied python executable
+        current = os.path.join(self.interpreter.prefix, "Python")
+        fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size)
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(CPython2macOsFramework, cls).sources(interpreter):
+            yield src
+
+        # landmark for exec_prefix
+        name = "lib-dynload"
+        yield PathRefToDest(Path(interpreter.system_stdlib) / name, dest=cls.to_stdlib)
+
+        # this must symlink to the host prefix Python
+        marker = Path(interpreter.prefix) / "Python"
+        ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True)
+        yield ref
+
+    @classmethod
+    def _executables(cls, interpreter):
+        for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter):
+            # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
+            # stub executable in ${sys.prefix}/bin.
+            # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
+            fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
+            yield fixed_host_exe, targets
+
+
+def fix_mach_o(exe, current, new, max_size):
+    """
+    https://en.wikipedia.org/wiki/Mach-O
+
+    Mach-O, short for Mach object file format, is a file format for executables, object code, shared libraries,
+    dynamically-loaded code, and core dumps. A replacement for the a.out format, Mach-O offers more extensibility and
+    faster access to information in the symbol table.
+
+    Each Mach-O file is made up of one Mach-O header, followed by a series of load commands, followed by one or more
+    segments, each of which contains between 0 and 255 sections. Mach-O uses the REL relocation format to handle
+    references to symbols. When looking up symbols Mach-O uses a two-level namespace that encodes each symbol into an
+    'object/symbol name' pair that is then linearly searched for by first the object and then the symbol name.
+
+    The basic structure—a list of variable-length "load commands" that reference pages of data elsewhere in the file—was
+    also used in the executable file format for Accent. The Accent file format was in turn, based on an idea from Spice
+    Lisp.
+
+    With the introduction of Mac OS X 10.6 platform the Mach-O file underwent a significant modification that causes
+    binaries compiled on a computer running 10.6 or later to be (by default) executable only on computers running Mac
+    OS X 10.6 or later. The difference stems from load commands that the dynamic linker, in previous Mac OS X versions,
+    does not understand. Another significant change to the Mach-O format is the change in how the Link Edit tables
+    (found in the __LINKEDIT section) function. In 10.6 these new Link Edit tables are compressed by removing unused and
+    unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format.
+    """
+    try:
+        logging.debug(u"change Mach-O for %s from %s to %s", ensure_text(exe), current, ensure_text(new))
+        _builtin_change_mach_o(max_size)(exe, current, new)
+    except Exception as e:
+        logging.warning("Could not call _builtin_change_mac_o: %s. " "Trying to call install_name_tool instead.", e)
+        try:
+            cmd = ["install_name_tool", "-change", current, new, exe]
+            subprocess.check_call(cmd)
+        except Exception:
+            logging.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed")
+            raise
+
+
+def _builtin_change_mach_o(maxint):
+    MH_MAGIC = 0xFEEDFACE
+    MH_CIGAM = 0xCEFAEDFE
+    MH_MAGIC_64 = 0xFEEDFACF
+    MH_CIGAM_64 = 0xCFFAEDFE
+    FAT_MAGIC = 0xCAFEBABE
+    BIG_ENDIAN = ">"
+    LITTLE_ENDIAN = "<"
+    LC_LOAD_DYLIB = 0xC
+
+    class FileView(object):
+        """A proxy for file-like objects that exposes a given view of a file. Modified from macholib."""
+
+        def __init__(self, file_obj, start=0, size=maxint):
+            if isinstance(file_obj, FileView):
+                self._file_obj = file_obj._file_obj
+            else:
+                self._file_obj = file_obj
+            self._start = start
+            self._end = start + size
+            self._pos = 0
+
+        def __repr__(self):
+            return "<fileview [{:d}, {:d}] {!r}>".format(self._start, self._end, self._file_obj)
+
+        def tell(self):
+            return self._pos
+
+        def _checkwindow(self, seek_to, op):
+            if not (self._start <= seek_to <= self._end):
+                msg = "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end)
+                raise IOError(msg)
+
+        def seek(self, offset, whence=0):
+            seek_to = offset
+            if whence == os.SEEK_SET:
+                seek_to += self._start
+            elif whence == os.SEEK_CUR:
+                seek_to += self._start + self._pos
+            elif whence == os.SEEK_END:
+                seek_to += self._end
+            else:
+                raise IOError("Invalid whence argument to seek: {!r}".format(whence))
+            self._checkwindow(seek_to, "seek")
+            self._file_obj.seek(seek_to)
+            self._pos = seek_to - self._start
+
+        def write(self, content):
+            here = self._start + self._pos
+            self._checkwindow(here, "write")
+            self._checkwindow(here + len(content), "write")
+            self._file_obj.seek(here, os.SEEK_SET)
+            self._file_obj.write(content)
+            self._pos += len(content)
+
+        def read(self, size=maxint):
+            assert size >= 0
+            here = self._start + self._pos
+            self._checkwindow(here, "read")
+            size = min(size, self._end - here)
+            self._file_obj.seek(here, os.SEEK_SET)
+            read_bytes = self._file_obj.read(size)
+            self._pos += len(read_bytes)
+            return read_bytes
+
+    def read_data(file, endian, num=1):
+        """Read a given number of 32-bits unsigned integers from the given file with the given endianness."""
+        res = struct.unpack(endian + "L" * num, file.read(num * 4))
+        if len(res) == 1:
+            return res[0]
+        return res
+
+    def mach_o_change(at_path, what, value):
+        """Replace a given name (what) in any LC_LOAD_DYLIB command found in the given binary with a new name (value),
+         provided it's shorter."""
+
+        def do_macho(file, bits, endian):
+            # Read Mach-O header (the magic number is assumed read by the caller)
+            cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6)
+            # 64-bits header has one more field.
+            if bits == 64:
+                read_data(file, endian)
+            # The header is followed by n commands
+            for _ in range(n_commands):
+                where = file.tell()
+                # Read command header
+                cmd, cmd_size = read_data(file, endian, 2)
+                if cmd == LC_LOAD_DYLIB:
+                    # The first data field in LC_LOAD_DYLIB commands is the offset of the name, starting from the
+                    # beginning of the  command.
+                    name_offset = read_data(file, endian)
+                    file.seek(where + name_offset, os.SEEK_SET)
+                    # Read the NUL terminated string
+                    load = file.read(cmd_size - name_offset).decode()
+                    load = load[: load.index("\0")]
+                    # If the string is what is being replaced, overwrite it.
+                    if load == what:
+                        file.seek(where + name_offset, os.SEEK_SET)
+                        file.write(value.encode() + b"\0")
+                # Seek to the next command
+                file.seek(where + cmd_size, os.SEEK_SET)
+
+        def do_file(file, offset=0, size=maxint):
+            file = FileView(file, offset, size)
+            # Read magic number
+            magic = read_data(file, BIG_ENDIAN)
+            if magic == FAT_MAGIC:
+                # Fat binaries contain nfat_arch Mach-O binaries
+                n_fat_arch = read_data(file, BIG_ENDIAN)
+                for _ in range(n_fat_arch):
+                    # Read arch header
+                    cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5)
+                    do_file(file, offset, size)
+            elif magic == MH_MAGIC:
+                do_macho(file, 32, BIG_ENDIAN)
+            elif magic == MH_CIGAM:
+                do_macho(file, 32, LITTLE_ENDIAN)
+            elif magic == MH_MAGIC_64:
+                do_macho(file, 64, BIG_ENDIAN)
+            elif magic == MH_CIGAM_64:
+                do_macho(file, 64, LITTLE_ENDIAN)
+
+        assert len(what) >= len(value)
+
+        with open(at_path, "r+b") as f:
+            do_file(f)
+
+    return mach_o_change

src/virtualenv/create/via_global_ref/builtin/python2/python2.py

@@ -32,7 +32,8 @@ def create(self):
         else:
             custom_site_text = custom_site.read_text()
         expected = json.dumps([os.path.relpath(ensure_text(str(i)), ensure_text(str(site_py))) for i in self.libs])
-        site_py.write_text(custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected))
+        custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)
+        site_py.write_text(custom_site_text)
 
     @classmethod
     def sources(cls, interpreter):