From 71ffc7184d07852d6746747d4509ae2c8cc3e8ec Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 28 Dec 2019 13:25:47 +0000 Subject: [PATCH 01/18] creator unicode support Signed-off-by: Bernat Gabor --- docs/conf.py | 4 +-- setup.cfg | 2 +- src/virtualenv/activation/bash/__init__.py | 2 +- src/virtualenv/activation/batch/__init__.py | 2 +- src/virtualenv/activation/cshell/__init__.py | 2 +- src/virtualenv/activation/fish/__init__.py | 2 +- .../activation/powershell/__init__.py | 2 +- src/virtualenv/activation/python/__init__.py | 2 +- src/virtualenv/activation/xonosh/__init__.py | 2 +- src/virtualenv/config/ini.py | 3 +-- src/virtualenv/info.py | 3 ++- .../interpreters/create/cpython/common.py | 3 +-- .../interpreters/create/cpython/cpython2.py | 3 +-- .../interpreters/create/cpython/cpython3.py | 3 +-- src/virtualenv/interpreters/create/creator.py | 8 +++--- .../interpreters/discovery/builtin.py | 3 +-- src/virtualenv/pyenv_cfg.py | 4 ++- src/virtualenv/seed/embed/link_app_data.py | 2 +- src/virtualenv/seed/embed/wheels/acquire.py | 2 +- src/virtualenv/util.py | 27 ++++++++++++++----- tests/conftest.py | 8 +++++- .../unit/interpreters/create/test_creator.py | 13 ++++----- .../discovery/windows/test_windows_pep514.py | 3 ++- 23 files changed, 62 insertions(+), 43 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c0a38a9c4..988f93669 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,8 +31,8 @@ def generate_draft_news(): env = os.environ.copy() env["PATH"] += os.pathsep.join([os.path.dirname(sys.executable)] + env["PATH"].split(os.pathsep)) changelog = subprocess.check_output( - ["towncrier", "--draft", "--version", "DRAFT"], cwd=str(ROOT_SRC_TREE_DIR), env=env - ).decode("utf-8") + ["towncrier", "--draft", "--version", "DRAFT"], cwd=str(ROOT_SRC_TREE_DIR), env=env, universal_newlines=True + ) if "No significant changes" in changelog: content = "" else: diff --git a/setup.cfg b/setup.cfg index 9bd5321b3..9a6c4d855 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,9 +42,9 @@ zip_safe = True python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* install_requires = six >= 1.12.0, < 2 - pathlib2 >= 2.3.3, < 3 appdirs >= 1.4.3 entrypoints >= 0.3, <1 + pathlib2 >= 2.3.3, < 3; python_version < '3.4' distlib >= 0.3.0, <1; sys.platform == 'win32' [options.packages.find] where = src diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index fd5374149..7c85564a8 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/batch/__init__.py b/src/virtualenv/activation/batch/__init__.py index 4e4b83965..2c3da4431 100644 --- a/src/virtualenv/activation/batch/__init__.py +++ b/src/virtualenv/activation/batch/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/cshell/__init__.py b/src/virtualenv/activation/cshell/__init__.py index e818edeeb..6a96d5676 100644 --- a/src/virtualenv/activation/cshell/__init__.py +++ b/src/virtualenv/activation/cshell/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 0e544068b..0dfe63c8d 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/powershell/__init__.py b/src/virtualenv/activation/powershell/__init__.py index b5b0a7527..a3c6f9e6c 100644 --- a/src/virtualenv/activation/powershell/__init__.py +++ b/src/virtualenv/activation/powershell/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/python/__init__.py b/src/virtualenv/activation/python/__init__.py index 1d73e9969..36b2b2cbb 100644 --- a/src/virtualenv/activation/python/__init__.py +++ b/src/virtualenv/activation/python/__init__.py @@ -3,7 +3,7 @@ import json import os -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/activation/xonosh/__init__.py b/src/virtualenv/activation/xonosh/__init__.py index ceb534057..94546ff9e 100644 --- a/src/virtualenv/activation/xonosh/__init__.py +++ b/src/virtualenv/activation/xonosh/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from pathlib2 import Path +from virtualenv.util import Path from ..via_template import ViaTemplateActivator diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index 9c86e48b5..4acb70ffc 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -3,9 +3,8 @@ import logging import os -from pathlib2 import Path - from virtualenv.info import PY3, get_default_config_dir +from virtualenv.util import Path from .convert import convert diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index 149f852ea..294110eba 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -3,7 +3,8 @@ import sys from appdirs import user_config_dir, user_data_dir -from pathlib2 import Path + +from virtualenv.util import Path IS_PYPY = hasattr(sys, "pypy_version_info") PY3 = sys.version_info[0] == 3 diff --git a/src/virtualenv/interpreters/create/cpython/common.py b/src/virtualenv/interpreters/create/cpython/common.py index d05d47bae..89a91c6df 100644 --- a/src/virtualenv/interpreters/create/cpython/common.py +++ b/src/virtualenv/interpreters/create/cpython/common.py @@ -4,10 +4,9 @@ from os import X_OK, access, chmod import six -from pathlib2 import Path from virtualenv.interpreters.create.via_global_ref import ViaGlobalRef -from virtualenv.util import copy, ensure_dir, symlink +from virtualenv.util import Path, copy, ensure_dir, symlink @six.add_metaclass(abc.ABCMeta) diff --git a/src/virtualenv/interpreters/create/cpython/cpython2.py b/src/virtualenv/interpreters/create/cpython/cpython2.py index fbaee889e..7c7915882 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython2.py +++ b/src/virtualenv/interpreters/create/cpython/cpython2.py @@ -3,9 +3,8 @@ import abc import six -from pathlib2 import Path -from virtualenv.util import copy +from virtualenv.util import Path, copy from .common import CPython, CPythonPosix, CPythonWindows diff --git a/src/virtualenv/interpreters/create/cpython/cpython3.py b/src/virtualenv/interpreters/create/cpython/cpython3.py index 3249306c8..b14ce0a86 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython3.py +++ b/src/virtualenv/interpreters/create/cpython/cpython3.py @@ -3,9 +3,8 @@ import abc import six -from pathlib2 import Path -from virtualenv.util import copy +from virtualenv.util import Path, copy from .common import CPython, CPythonPosix, CPythonWindows diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index 872d8d10b..e5bd8b6b9 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -7,12 +7,12 @@ from abc import ABCMeta, abstractmethod from argparse import ArgumentTypeError -from pathlib2 import Path +import six from six import add_metaclass from virtualenv.info import IS_WIN from virtualenv.pyenv_cfg import PyEnvCfg -from virtualenv.util import run_cmd +from virtualenv.util import Path, run_cmd from virtualenv.version import __version__ HERE = Path(__file__).absolute().parent @@ -104,7 +104,7 @@ def set_pyenv_cfg(self): @property def env_name(self): - return self.dest_dir.parts[-1] + return six.ensure_text(self.dest_dir.parts[-1]) @property def bin_name(self): @@ -139,7 +139,7 @@ def debug_script(self): def get_env_debug_info(env_exe, debug_script): cmd = [str(env_exe), str(debug_script)] - logging.debug(" ".join(cmd)) + logging.debug(" ".join(six.ensure_text(i) for i in cmd)) env = os.environ.copy() env.pop("PYTHONPATH", None) code, out, err = run_cmd(cmd) diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py index 1dc80c030..dfccb5437 100644 --- a/src/virtualenv/interpreters/discovery/builtin.py +++ b/src/virtualenv/interpreters/discovery/builtin.py @@ -4,9 +4,8 @@ import os import sys -from pathlib2 import Path - from virtualenv.info import IS_WIN +from virtualenv.util import Path from .discover import Discover from .py_info import CURRENT, PythonInfo diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index 06a8c8162..b6381b7e1 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -2,6 +2,8 @@ import logging +import six + class PyEnvCfg(object): def __init__(self, content, path): @@ -29,7 +31,7 @@ def _read_values(path): def write(self): with open(str(self.path), "wt") as file_handler: - logging.debug("write %s", self.path) + logging.debug("write %s", six.ensure_text(str(self.path))) for key, value in self.content.items(): line = "{} = {}".format(key, value) logging.debug("\t%s", line) diff --git a/src/virtualenv/seed/embed/link_app_data.py b/src/virtualenv/seed/embed/link_app_data.py index c400da718..b38814467 100644 --- a/src/virtualenv/seed/embed/link_app_data.py +++ b/src/virtualenv/seed/embed/link_app_data.py @@ -9,10 +9,10 @@ from shutil import copytree from textwrap import dedent -from pathlib2 import Path from six import PY3 from virtualenv.info import get_default_data_dir +from virtualenv.util import Path from .base_embed import BaseEmbed from .wheels.acquire import get_wheel diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index a04c1b89a..fcfb26357 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -5,7 +5,7 @@ from collections import defaultdict from shutil import copy2 -from pathlib2 import Path +from virtualenv.util import Path from . import BUNDLE_SUPPORT, MAX diff --git a/src/virtualenv/util.py b/src/virtualenv/util.py index 09c38e67e..30d4010b9 100644 --- a/src/virtualenv/util.py +++ b/src/virtualenv/util.py @@ -9,11 +9,16 @@ import six +if six.PY3: + from pathlib import Path +else: + from pathlib2 import Path + def ensure_dir(path): if not path.exists(): - logging.debug("created %s", path) - makedirs(six.text_type(path)) + logging.debug("created %s", six.ensure_text(str(path))) + makedirs(str(path)) HAS_SYMLINK = hasattr(os, "symlink") @@ -30,16 +35,21 @@ def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): if not dst.is_symlink(): # can't link to itself! if relative_symlinks_ok: assert src.parent == dst.parent - os.symlink(src.name, six.text_type(dst)) + os.symlink(src.name, str(dst)) else: - os.symlink(six.text_type(src), six.text_type(dst)) + os.symlink(str(src), str(dst)) except OSError as exception: - logging.warning("symlink failed %r, for %s to %s, will try copy", exception, src, dst) + logging.warning( + "symlink failed %r, for %s to %s, will try copy", + exception, + six.ensure_text(str(src)), + six.ensure_text(str(dst)), + ) do_copy = True if do_copy: copier = shutil.copy2 if src.is_file() else shutil.copytree - copier(six.text_type(src), six.text_type(dst)) - logging.debug("%s %s to %s", "copy" if do_copy else "symlink", src, dst) + copier(str(src), str(dst)) + logging.debug("%s %s to %s", "copy" if do_copy else "symlink", six.ensure_text(str(src)), six.ensure_text(str(dst))) def run_cmd(cmd): @@ -56,3 +66,6 @@ def run_cmd(cmd): symlink = partial(symlink_or_copy, False) copy = partial(symlink_or_copy, True) + + +__all__ = ("Path", "symlink", "copy", "run_cmd", "ensure_dir") diff --git a/tests/conftest.py b/tests/conftest.py index 515141093..79a962c16 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals import os @@ -7,9 +8,9 @@ import coverage import pytest -from pathlib2 import Path from virtualenv.interpreters.discovery.py_info import PythonInfo +from virtualenv.util import Path @pytest.fixture(scope="session") @@ -220,3 +221,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): @pytest.fixture(scope="session") def is_inside_ci(): yield "CI_RUN" in os.environ + + +@pytest.fixture(scope="session") +def special_char_name(): + return "πŸš’Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡" diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 5e35748d5..7aef9001a 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -7,7 +7,6 @@ import pytest import six -from pathlib2 import Path from virtualenv.__main__ import run from virtualenv.interpreters.create.creator import DEBUG_SCRIPT, get_env_debug_info @@ -15,6 +14,7 @@ from virtualenv.interpreters.discovery.py_info import CURRENT, PythonInfo from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.run import run_via_cli, session_via_cli +from virtualenv.util import Path def test_os_path_sep_not_allowed(tmp_path, capsys): @@ -80,8 +80,9 @@ def cleanup_sys_path(path): @pytest.mark.parametrize( "use_venv", [False, True] if six.PY3 else [False], ids=["no_venv", "venv"] if six.PY3 else ["no_venv"] ) -def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env): - cmd = ["-v", "-v", "-p", str(python), str(tmp_path), "--without-pip", "--activators", ""] +def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, special_char_name): + dest = tmp_path / special_char_name + cmd = ["-v", "-v", "-p", str(python), str(dest), "--without-pip", "--activators", ""] if global_access: cmd.append("--system-site-packages") if use_venv: @@ -91,7 +92,7 @@ def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env) for site_package in result.creator.site_packages: content = list(site_package.iterdir()) assert not content, "\n".join(str(i) for i in content) - assert result.creator.env_name == tmp_path.name + assert result.creator.env_name == special_char_name sys_path = cleanup_sys_path(result.creator.debug["sys"]["path"]) system_sys_path = cleanup_sys_path(SYSTEM["sys"]["path"]) our_paths = set(sys_path) - set(system_sys_path) @@ -101,7 +102,7 @@ def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env) assert len(our_paths) >= 1, our_paths_repr # ensure all additional paths are related to the virtual environment for path in our_paths: - assert str(path).startswith(str(tmp_path)), path + assert str(path).startswith(str(dest)), path # ensure there's at least a site-packages folder as part of the virtual environment added assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr @@ -116,7 +117,7 @@ def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env) break def list_to_str(iterable): - return [str(i) for i in iterable] + return [six.ensure_text(str(i)) for i in iterable] assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) else: diff --git a/tests/unit/interpreters/discovery/windows/test_windows_pep514.py b/tests/unit/interpreters/discovery/windows/test_windows_pep514.py index 81ae829d8..0a2807785 100644 --- a/tests/unit/interpreters/discovery/windows/test_windows_pep514.py +++ b/tests/unit/interpreters/discovery/windows/test_windows_pep514.py @@ -7,7 +7,8 @@ import pytest import six -from pathlib2 import Path + +from virtualenv.util import Path @pytest.mark.skipif(sys.platform != "win32", reason="Windows registry only on Windows platform") From 6110c6f201ff42ce556669bb296e7520a8c5f3a2 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 28 Dec 2019 18:21:47 +0000 Subject: [PATCH 02/18] activator support Signed-off-by: Bernat Gabor --- src/virtualenv/activation/bash/activate.sh | 2 +- src/virtualenv/activation/cshell/activate.csh | 6 ++-- src/virtualenv/activation/fish/activate.fish | 26 +++++++------- .../activation/python/activate_this.py | 3 ++ src/virtualenv/activation/via_template.py | 6 ++-- src/virtualenv/interpreters/create/creator.py | 16 ++++++--- src/virtualenv/pyenv_cfg.py | 2 +- tests/unit/activation/conftest.py | 35 +++++++++++++------ tests/unit/activation/test_fish.py | 2 +- .../unit/activation/test_python_activator.py | 24 +++++++++---- 10 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/virtualenv/activation/bash/activate.sh b/src/virtualenv/activation/bash/activate.sh index d9b878154..19bf552bd 100644 --- a/src/virtualenv/activation/bash/activate.sh +++ b/src/virtualenv/activation/bash/activate.sh @@ -46,7 +46,7 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VIRTUAL_ENV__" +VIRTUAL_ENV='__VIRTUAL_ENV__' export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" diff --git a/src/virtualenv/activation/cshell/activate.csh b/src/virtualenv/activation/cshell/activate.csh index c4a6d584c..72b2cf8ef 100644 --- a/src/virtualenv/activation/cshell/activate.csh +++ b/src/virtualenv/activation/cshell/activate.csh @@ -10,15 +10,15 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VIRTUAL_ENV__" +setenv VIRTUAL_ENV '__VIRTUAL_ENV__' set _OLD_VIRTUAL_PATH="$PATH:q" setenv PATH "$VIRTUAL_ENV:q/__BIN_NAME__:$PATH:q" -if ("__VIRTUAL_PROMPT__" != "") then - set env_name = "__VIRTUAL_PROMPT__" +if ('__VIRTUAL_PROMPT__' != "") then + set env_name = '__VIRTUAL_PROMPT__' else set env_name = '('"$VIRTUAL_ENV:t:q"') ' endif diff --git a/src/virtualenv/activation/fish/activate.fish b/src/virtualenv/activation/fish/activate.fish index 4e2976864..74ec9cc77 100644 --- a/src/virtualenv/activation/fish/activate.fish +++ b/src/virtualenv/activation/fish/activate.fish @@ -11,23 +11,23 @@ function _bashify_path -d "Converts a fish path to something bash can recognize" end function _fishify_path -d "Converts a bash path to something fish can recognize" - echo $argv | tr ':' '\n' + echo $argv | string replace ':' '\n' end function deactivate -d 'Exit virtualenv mode and return to the normal environment.' # reset old environment variables if test -n "$_OLD_VIRTUAL_PATH" # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling - if test (echo $FISH_VERSION | tr "." "\n")[1] -lt 3 - set -gx PATH (_fishify_path $_OLD_VIRTUAL_PATH) + if test (echo $FISH_VERSION | head -c 1) -lt 3 + set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH") else - set -gx PATH $_OLD_VIRTUAL_PATH + set -gx PATH "$_OLD_VIRTUAL_PATH" end set -e _OLD_VIRTUAL_PATH end if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME" set -e _OLD_VIRTUAL_PYTHONHOME end @@ -57,15 +57,15 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VIRTUAL_ENV__" +set -gx VIRTUAL_ENV '__VIRTUAL_ENV__' # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling -if test (echo $FISH_VERSION | tr "." "\n")[1] -lt 3 - set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) +if test (echo $FISH_VERSION | head -c 1) -lt 3 + set -gx _OLD_VIRTUAL_PATH (_bashify_path "$PATH") else - set -gx _OLD_VIRTUAL_PATH $PATH + set -gx _OLD_VIRTUAL_PATH "$PATH" end -set -gx PATH "$VIRTUAL_ENV/__BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV"'/__BIN_NAME__' "$PATH" # Unset `$PYTHONHOME` if set. if set -q PYTHONHOME @@ -87,10 +87,10 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" # Prompt override provided? # If not, just prepend the environment name. - if test -n "__VIRTUAL_PROMPT__" - printf '%s%s' "__VIRTUAL_PROMPT__" (set_color normal) + if test -n '__VIRTUAL_PROMPT__' + printf '%s%s' '__VIRTUAL_PROMPT__' (set_color normal) else - printf '%s(%s) ' (set_color normal) (basename "$VIRTUAL_ENV") + printf '%s(%s) ' (set_color normal) (basename '$VIRTUAL_ENV') end # Restore the original $status diff --git a/src/virtualenv/activation/python/activate_this.py b/src/virtualenv/activation/python/activate_this.py index fc8d449e7..dc1e977ab 100644 --- a/src/virtualenv/activation/python/activate_this.py +++ b/src/virtualenv/activation/python/activate_this.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Activate virtualenv for current interpreter: Use exec(open(this_file).read(), {'__file__': this_file}). @@ -33,6 +34,8 @@ ''' for site_package in json.loads(site_packages): + if sys.version_info[0] == 2: + site_package = site_package.encode('utf-8') path = os.path.realpath(os.path.join(os.path.dirname(__file__), site_package)) site.addsitedir(path) # fmt: on diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 864454f95..34b06fa8f 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -24,9 +24,9 @@ def generate(self, creator): def replacements(self, creator, dest_folder): return { "__VIRTUAL_PROMPT__": "" if self.flag_prompt is None else self.flag_prompt, - "__VIRTUAL_ENV__": str(creator.dest_dir), - "__VIRTUAL_NAME__": str(creator.env_name), - "__BIN_NAME__": str(creator.bin_name), + "__VIRTUAL_ENV__": six.ensure_text(str(creator.dest_dir)), + "__VIRTUAL_NAME__": creator.env_name, + "__BIN_NAME__": six.ensure_text(str(creator.bin_name)), "__PATH_SEP__": os.pathsep, } diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index e5bd8b6b9..c8aa3d83f 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -4,6 +4,7 @@ import logging import os import shutil +import sys from abc import ABCMeta, abstractmethod from argparse import ArgumentTypeError @@ -47,17 +48,22 @@ def add_parser_arguments(cls, parser, interpreter): help="Give the virtual environment access to the system site-packages dir.", ) - def validate_dest_dir(value): + def validate_dest_dir(raw_value): """No path separator in the path and must be write-able""" - if os.pathsep in value: + if os.pathsep in raw_value: raise ArgumentTypeError( "destination {!r} must not contain the path separator ({}) as this would break " - "the activation scripts".format(value, os.pathsep) + "the activation scripts".format(raw_value, os.pathsep) ) - value = Path(value) + value = Path(raw_value) if value.exists() and value.is_file(): raise ArgumentTypeError("the destination {} already exists and is a file".format(value)) - value = dest = value.resolve() + if (3, 3) <= sys.version_info <= (3, 6): + # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation + dest = Path(os.path.realpath(raw_value)) + else: + dest = value.resolve() + value = dest while dest: if dest.exists(): if os.access(str(dest), os.W_OK): diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index b6381b7e1..6dd82c09c 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -35,7 +35,7 @@ def write(self): for key, value in self.content.items(): line = "{} = {}".format(key, value) logging.debug("\t%s", line) - file_handler.write(line) + file_handler.write(six.ensure_str(line)) file_handler.write("\n") def refresh(self): diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 13eb7d369..e91c1b7c4 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -11,6 +11,7 @@ import six from virtualenv.run import run_via_cli +from virtualenv.util import Path class ActivationTester(object): @@ -28,7 +29,13 @@ def __init__(self, of_class, session, cmd, activate_script, extension): def get_version(self, raise_on_fail): # locally we disable, so that contributors don't need to have everything setup try: - return subprocess.check_output(self._version_cmd, universal_newlines=True) + process = subprocess.Popen( + self._version_cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + out, err = process.communicate() + if out: + return out + return err except Exception as exception: if raise_on_fail: raise @@ -43,10 +50,12 @@ def __call__(self, monkeypatch, tmp_path): invoke, env = self._invoke_script + [str(test_script)], self.env(tmp_path) try: - raw = subprocess.check_output(invoke, universal_newlines=True, stderr=subprocess.STDOUT, env=env) + _raw = subprocess.check_output(invoke, universal_newlines=True, stderr=subprocess.STDOUT, env=env) + raw = "\n{}".format(six.ensure_text(_raw)) except subprocess.CalledProcessError as exception: - assert not exception.returncode, exception.output + assert not exception.returncode, six.ensure_text(exception.output) return + out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split("\n") self.assert_output(out, raw, tmp_path) return env, activate_script @@ -116,12 +125,14 @@ def print_os_env_var(self, var): return self.python_cmd("import os; print(os.environ.get({}, None))".format(val)) def activate_call(self, script): - return "{} {}".format(self.quote(str(self.activate_cmd)), self.quote(str(script))).strip() + cmd = self.quote(six.ensure_text(str(self.activate_cmd))) + scr = self.quote(six.ensure_text(str(script))) + return "{} {}".format(cmd, scr).strip() @staticmethod def norm_path(path): # python may return Windows short paths, normalize - path = realpath(str(path)) + path = realpath(six.ensure_text(str(path)) if isinstance(path, Path) else path) if sys.platform == "win32": from ctypes import create_unicode_buffer, windll @@ -150,7 +161,7 @@ def __call__(self, monkeypatch, tmp_path): ) out, err = process.communicate() assert process.returncode - assert self.non_source_fail_message in err + assert self.non_source_fail_message in six.ensure_text(err) @pytest.fixture(scope="session") @@ -164,16 +175,16 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") -def activation_python(tmp_path_factory): - dest = tmp_path_factory.mktemp("a") - session = run_via_cli(["--seed", "none", str(dest)]) +def activation_python(tmp_path_factory, special_char_name): + dest = tmp_path_factory.mktemp(six.ensure_str(special_char_name)) + session = run_via_cli(["--seed", "none", str(dest), "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" pydoc_test.write_text('"""This is pydoc_test.py"""') return session @pytest.fixture() -def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci): +def activation_tester(activation_python, monkeypatch, tmp_path, special_char_name, is_inside_ci): def _tester(tester_class): tester = tester_class(activation_python) if not tester.of_class.supports(activation_python.creator.interpreter): @@ -181,6 +192,8 @@ def _tester(tester_class): version = tester.get_version(raise_on_fail=is_inside_ci) if not isinstance(version, six.string_types): pytest.skip(msg=six.text_type(version)) - return tester(monkeypatch, tmp_path) + folder = tmp_path / special_char_name + folder.mkdir() + return tester(monkeypatch, folder) return _tester diff --git a/tests/unit/activation/test_fish.py b/tests/unit/activation/test_fish.py index bf154649c..4b8533895 100644 --- a/tests/unit/activation/test_fish.py +++ b/tests/unit/activation/test_fish.py @@ -3,7 +3,7 @@ from virtualenv.activation import FishActivator -def test_csh(activation_tester_class, activation_tester): +def test_fish(activation_tester_class, activation_tester): class Fish(activation_tester_class): def __init__(self, session): super(Fish, self).__init__(FishActivator, session, "fish", "activate.fish", "fish") diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index 6320e82cb..4070c45f3 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -4,6 +4,8 @@ import os import sys +import six + from virtualenv.activation import PythonActivator @@ -28,8 +30,9 @@ def env(self, tmp_path): def _get_test_lines(self, activate_script): raw = inspect.getsource(self.activate_this_test) - raw = raw.replace("__FILENAME__", str(activate_script)) - return [i.lstrip() for i in raw.splitlines()[2:]] + return ["# -*- coding: utf-8 -*-"] + [ + i[12:] for i in raw.replace("__FILENAME__", six.ensure_text(str(activate_script))).splitlines()[2:] + ] # noinspection PyUnresolvedReferences @staticmethod @@ -41,7 +44,9 @@ def activate_this_test(): print(os.environ.get("PATH")) print(os.pathsep.join(sys.path)) file_at = r"__FILENAME__" - exec(open(file_at).read(), {"__file__": file_at}) + with open(file_at, "rt") as file_handler: + content = file_handler.read() + exec(content, {"__file__": file_at}) print(os.environ.get("VIRTUAL_ENV")) print(os.environ.get("PATH")) print(os.pathsep.join(sys.path)) @@ -56,19 +61,24 @@ def assert_output(self, out, raw, tmp_path): prev_path = out[1].split(os.path.pathsep) prev_sys_path = out[2].split(os.path.pathsep) - assert out[3] == str(self._creator.dest_dir) # VIRTUAL_ENV now points to the virtual env folder + assert out[3] == six.ensure_text( + str(self._creator.dest_dir) + ) # VIRTUAL_ENV now points to the virtual env folder new_path = out[4].split(os.pathsep) # PATH now starts with bin path of current - assert ([str(self._creator.bin_dir)] + prev_path) == new_path + assert ([six.ensure_text(str(self._creator.bin_dir))] + prev_path) == new_path # sys path contains the site package at its start new_sys_path = out[5].split(os.path.pathsep) - assert ([str(i) for i in self._creator.site_packages] + prev_sys_path) == new_sys_path + assert ([six.ensure_text(str(i)) for i in self._creator.site_packages] + prev_sys_path) == new_sys_path # manage to import from activate site package assert self.norm_path(out[6]) == self.norm_path(self._creator.site_packages[0] / "pydoc_test.py") def non_source_activate(self, activate_script): - return self._invoke_script + ["-c", 'exec(open(r"{}").read())'.format(activate_script)] + return self._invoke_script + [ + "-c", + 'exec(open(r"{}").read())'.format(six.ensure_text(str(activate_script))), + ] activation_tester(Python) From 135a6dbef296271e796cd7056c1fbd9b5d764cfd Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 29 Dec 2019 22:04:28 +0000 Subject: [PATCH 03/18] fix --- src/virtualenv/activation/fish/activate.fish | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/virtualenv/activation/fish/activate.fish b/src/virtualenv/activation/fish/activate.fish index 74ec9cc77..770fb0d4f 100644 --- a/src/virtualenv/activation/fish/activate.fish +++ b/src/virtualenv/activation/fish/activate.fish @@ -11,7 +11,7 @@ function _bashify_path -d "Converts a fish path to something bash can recognize" end function _fishify_path -d "Converts a bash path to something fish can recognize" - echo $argv | string replace ':' '\n' + echo $argv | tr ':' '\n' end function deactivate -d 'Exit virtualenv mode and return to the normal environment.' @@ -61,11 +61,11 @@ set -gx VIRTUAL_ENV '__VIRTUAL_ENV__' # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling if test (echo $FISH_VERSION | head -c 1) -lt 3 - set -gx _OLD_VIRTUAL_PATH (_bashify_path "$PATH") + set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) else set -gx _OLD_VIRTUAL_PATH "$PATH" end -set -gx PATH "$VIRTUAL_ENV"'/__BIN_NAME__' "$PATH" +set -gx PATH "$VIRTUAL_ENV"'/__BIN_NAME__' $PATH # Unset `$PYTHONHOME` if set. if set -q PYTHONHOME From a757d17182d6b696923c6bb737f7eb42051a7998 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 29 Dec 2019 22:07:24 +0000 Subject: [PATCH 04/18] add space --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 79a962c16..3bd9e975b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -225,4 +225,4 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): - return "πŸš’Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡" + return "πŸš’Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡ " From fc27997b094dd693ca14b900f82693e8b152d94c Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 29 Dec 2019 22:17:40 +0000 Subject: [PATCH 05/18] python3.4 support --- setup.cfg | 2 +- src/virtualenv/{util.py => util/__init__.py} | 6 +--- src/virtualenv/util/path.py | 30 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) rename src/virtualenv/{util.py => util/__init__.py} (96%) create mode 100644 src/virtualenv/util/path.py diff --git a/setup.cfg b/setup.cfg index 9a6c4d855..45694e863 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ where = src [options.extras_require] testing = pytest >= 4.0.0, <6 - coverage >= 5.0.1, <6 + coverage >= 4.5.1, <6 pytest-mock >= 1.12.1, <2 xonsh >= 0.9.13, <1; python_version > '3.4' docs = diff --git a/src/virtualenv/util.py b/src/virtualenv/util/__init__.py similarity index 96% rename from src/virtualenv/util.py rename to src/virtualenv/util/__init__.py index 30d4010b9..a2cb48bc8 100644 --- a/src/virtualenv/util.py +++ b/src/virtualenv/util/__init__.py @@ -9,10 +9,7 @@ import six -if six.PY3: - from pathlib import Path -else: - from pathlib2 import Path +from .path import Path def ensure_dir(path): @@ -67,5 +64,4 @@ def run_cmd(cmd): symlink = partial(symlink_or_copy, False) copy = partial(symlink_or_copy, True) - __all__ = ("Path", "symlink", "copy", "run_cmd", "ensure_dir") diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py new file mode 100644 index 000000000..d59b47770 --- /dev/null +++ b/src/virtualenv/util/path.py @@ -0,0 +1,30 @@ +import six + +if six.PY3: + from pathlib import Path + + if six.PY34: + BuiltinPath = Path + + class Path(type(BuiltinPath())): + def read_text(self, encoding=None, errors=None): + """ + Open the file in text mode, read it, and close the file. + """ + with self.open(mode="r", encoding=encoding, errors=errors) as f: + return f.read() + + def write_text(self, data, encoding=None, errors=None): + """ + Open the file in text mode, write to it, and close the file. + """ + if not isinstance(data, str): + raise TypeError("data must be str, not %s" % data.__class__.__name__) + with self.open(mode="w", encoding=encoding, errors=errors) as f: + return f.write(data) + + +else: + from pathlib2 import Path + +__all__ = ("Path",) From 8c143be069624cdbb9c1351765a5ae490f094ea6 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 29 Dec 2019 22:49:15 +0000 Subject: [PATCH 06/18] Windows fixes --- src/virtualenv/activation/via_template.py | 2 +- src/virtualenv/pyenv_cfg.py | 2 +- src/virtualenv/seed/embed/link_app_data.py | 4 ++-- src/virtualenv/util/path.py | 3 ++- tests/unit/activation/conftest.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 34b06fa8f..884c3a4dd 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,4 +35,4 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - (to_folder / template).write_text(text) + (to_folder / template).write_text(text, encoding='utf-8') diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index 6dd82c09c..acb854ad6 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -30,7 +30,7 @@ def _read_values(path): return content def write(self): - with open(str(self.path), "wt") as file_handler: + with open(str(self.path), "w", encoding='utf-8') as file_handler: logging.debug("write %s", six.ensure_text(str(self.path))) for key, value in self.content.items(): line = "{} = {}".format(key, value) diff --git a/src/virtualenv/seed/embed/link_app_data.py b/src/virtualenv/seed/embed/link_app_data.py index b38814467..0c60f100d 100644 --- a/src/virtualenv/seed/embed/link_app_data.py +++ b/src/virtualenv/seed/embed/link_app_data.py @@ -172,7 +172,7 @@ def create_console_entry_point(bin_dir, name, value, env_exe, creator): "{}-{}.{}".format(name, version.major, version.minor), ): exe = bin_dir / new_name - exe.write_text(content) + exe.write_text(content, encoding='utf-8') exe.chmod(0o755) result.append(exe) return result @@ -210,7 +210,7 @@ def _handle_file(of, base): record = dist_info / "RECORD" content = ("" if folder_linked else record.read_text()) + "\n".join(new_records) - record.write_text(content) + record.write_text(content, encoding='utf-8') def add_record_line(name): diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index d59b47770..1e69fa3f0 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -1,9 +1,10 @@ import six +import sys if six.PY3: from pathlib import Path - if six.PY34: + if sys.version_info[0:2] == (3, 4): BuiltinPath = Path class Path(type(BuiltinPath())): diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index e91c1b7c4..d05d248ff 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -78,7 +78,7 @@ def _generate_test_script(self, activate_script, tmp_path): commands = self._get_test_lines(activate_script) script = os.linesep.join(commands) test_script = tmp_path / "script.{}".format(self.extension) - test_script.write_text(script) + test_script.write_text(script, encoding='utf-8') return test_script def _get_test_lines(self, activate_script): From 5613294c3cdd5671e24ef2d4fb20caf3a7d5915e Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 15:38:21 +0000 Subject: [PATCH 07/18] some fixes --- src/virtualenv/activation/via_template.py | 2 +- src/virtualenv/pyenv_cfg.py | 4 ++-- src/virtualenv/seed/embed/link_app_data.py | 4 ++-- src/virtualenv/util/path.py | 3 ++- tests/conftest.py | 2 +- tests/unit/activation/conftest.py | 20 ++++++++------------ tests/unit/activation/test_batch.py | 3 ++- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 884c3a4dd..e7b3aadf4 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,4 +35,4 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - (to_folder / template).write_text(text, encoding='utf-8') + (to_folder / template).write_text(text, encoding="utf-8") diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index acb854ad6..fa3431fce 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -22,7 +22,7 @@ def from_file(cls, path): @staticmethod def _read_values(path): content = {} - for line in path.read_text().splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): equals_at = line.index("=") key = line[:equals_at].strip() value = line[equals_at + 1 :].strip() @@ -30,7 +30,7 @@ def _read_values(path): return content def write(self): - with open(str(self.path), "w", encoding='utf-8') as file_handler: + with open(str(self.path), "w", encoding="utf-8") as file_handler: logging.debug("write %s", six.ensure_text(str(self.path))) for key, value in self.content.items(): line = "{} = {}".format(key, value) diff --git a/src/virtualenv/seed/embed/link_app_data.py b/src/virtualenv/seed/embed/link_app_data.py index 0c60f100d..d660574a3 100644 --- a/src/virtualenv/seed/embed/link_app_data.py +++ b/src/virtualenv/seed/embed/link_app_data.py @@ -172,7 +172,7 @@ def create_console_entry_point(bin_dir, name, value, env_exe, creator): "{}-{}.{}".format(name, version.major, version.minor), ): exe = bin_dir / new_name - exe.write_text(content, encoding='utf-8') + exe.write_text(content, encoding="utf-8") exe.chmod(0o755) result.append(exe) return result @@ -210,7 +210,7 @@ def _handle_file(of, base): record = dist_info / "RECORD" content = ("" if folder_linked else record.read_text()) + "\n".join(new_records) - record.write_text(content, encoding='utf-8') + record.write_text(content, encoding="utf-8") def add_record_line(name): diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index 1e69fa3f0..89cd04e65 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -1,6 +1,7 @@ -import six import sys +import six + if six.PY3: from pathlib import Path diff --git a/tests/conftest.py b/tests/conftest.py index 3bd9e975b..d6bd4baf7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -225,4 +225,4 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): - return "πŸš’Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡ " + return "πŸš’ Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡" diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index d05d248ff..6e2b949f9 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -50,13 +50,13 @@ def __call__(self, monkeypatch, tmp_path): invoke, env = self._invoke_script + [str(test_script)], self.env(tmp_path) try: - _raw = subprocess.check_output(invoke, universal_newlines=True, stderr=subprocess.STDOUT, env=env) - raw = "\n{}".format(six.ensure_text(_raw)) + _raw = subprocess.check_output(invoke, stderr=subprocess.STDOUT, env=env) + raw = "\n{}".format(_raw.decode("utf-8")) except subprocess.CalledProcessError as exception: assert not exception.returncode, six.ensure_text(exception.output) return - out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split("\n") + out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split(os.linesep) self.assert_output(out, raw, tmp_path) return env, activate_script @@ -78,7 +78,7 @@ def _generate_test_script(self, activate_script, tmp_path): commands = self._get_test_lines(activate_script) script = os.linesep.join(commands) test_script = tmp_path / "script.{}".format(self.extension) - test_script.write_text(script, encoding='utf-8') + test_script.write_text(script, encoding="utf-8") return test_script def _get_test_lines(self, activate_script): @@ -153,15 +153,11 @@ def __init__(self, of_class, session, cmd, activate_script, extension, non_sourc def __call__(self, monkeypatch, tmp_path): env, activate_script = super(RaiseOnNonSourceCall, self).__call__(monkeypatch, tmp_path) process = subprocess.Popen( - self.non_source_activate(activate_script), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - universal_newlines=True, + self.non_source_activate(activate_script), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) out, err = process.communicate() assert process.returncode - assert self.non_source_fail_message in six.ensure_text(err) + assert self.non_source_fail_message in err.decode("utf-8") @pytest.fixture(scope="session") @@ -176,7 +172,7 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") def activation_python(tmp_path_factory, special_char_name): - dest = tmp_path_factory.mktemp(six.ensure_str(special_char_name)) + dest = tmp_path_factory.mktemp(six.ensure_str("env-{}-v".format(special_char_name))) session = run_via_cli(["--seed", "none", str(dest), "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" pydoc_test.write_text('"""This is pydoc_test.py"""') @@ -192,7 +188,7 @@ def _tester(tester_class): version = tester.get_version(raise_on_fail=is_inside_ci) if not isinstance(version, six.string_types): pytest.skip(msg=six.text_type(version)) - folder = tmp_path / special_char_name + folder = tmp_path / "test-{}-env".format(special_char_name) folder.mkdir() return tester(monkeypatch, folder) diff --git a/tests/unit/activation/test_batch.py b/tests/unit/activation/test_batch.py index 143943b1e..92b141b03 100644 --- a/tests/unit/activation/test_batch.py +++ b/tests/unit/activation/test_batch.py @@ -19,7 +19,8 @@ def __init__(self, session): self.pydoc_call = "call {}".format(self.pydoc_call) def _get_test_lines(self, activate_script): - return ["@echo off", ""] + super(Batch, self)._get_test_lines(activate_script) + # for BATCH utf-8 support need change the character code page to 650001 + return ["@echo off", "", "chcp 65001 1>NUL"] + super(Batch, self)._get_test_lines(activate_script) def quote(self, s): """double quotes needs to be single, and single need to be double""" From 184858c7604539391aa57c7064d9f34097ac8f01 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 16:14:42 +0000 Subject: [PATCH 08/18] fix powershell requires utf-16 --- tests/unit/activation/conftest.py | 3 ++- tests/unit/activation/test_powershell.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 6e2b949f9..1384cf501 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -25,6 +25,7 @@ def __init__(self, of_class, session, cmd, activate_script, extension): self.activate_cmd = "source" self.deactivate = "deactivate" self.pydoc_call = "pydoc -w pydoc_test" + self.script_encoding = "utf-8" def get_version(self, raise_on_fail): # locally we disable, so that contributors don't need to have everything setup @@ -78,7 +79,7 @@ def _generate_test_script(self, activate_script, tmp_path): commands = self._get_test_lines(activate_script) script = os.linesep.join(commands) test_script = tmp_path / "script.{}".format(self.extension) - test_script.write_text(script, encoding="utf-8") + test_script.write_text(script, encoding=self.script_encoding) return test_script def _get_test_lines(self, activate_script): diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py index e2edac4d3..88567eb42 100644 --- a/tests/unit/activation/test_powershell.py +++ b/tests/unit/activation/test_powershell.py @@ -14,11 +14,16 @@ def __init__(self, session): self._version_cmd = [cmd, "-c", "$PSVersionTable"] self._invoke_script = [cmd, "-ExecutionPolicy", "ByPass", "-File"] self.activate_cmd = "." + self.script_encoding = "utf-16" def quote(self, s): """powershell double double quote needed for quotes within single quotes""" return pipes.quote(s).replace('"', '""') + def _get_test_lines(self, activate_script): + # for BATCH utf-8 support need change the character code page to 650001 + return super(PowerShell, self)._get_test_lines(activate_script) + def invoke_script(self): return [self.cmd, "-File"] From cd6a73df15c89fcf3d45671468d4d9e439581d09 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 16:36:30 +0000 Subject: [PATCH 09/18] try to fix python2 windows Signed-off-by: Bernat Gabor --- src/virtualenv/pyenv_cfg.py | 6 +++--- src/virtualenv/util/path.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index fa3431fce..14860d562 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -30,13 +30,13 @@ def _read_values(path): return content def write(self): - with open(str(self.path), "w", encoding="utf-8") as file_handler: + with open(str(self.path), "wb") as file_handler: logging.debug("write %s", six.ensure_text(str(self.path))) for key, value in self.content.items(): line = "{} = {}".format(key, value) logging.debug("\t%s", line) - file_handler.write(six.ensure_str(line)) - file_handler.write("\n") + file_handler.write(line.encode("utf-8")) + file_handler.write(b"\n") def refresh(self): self.content = self._read_values(self.path) diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index 89cd04e65..3c6be2a5f 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -29,4 +29,8 @@ def write_text(self, data, encoding=None, errors=None): else: from pathlib2 import Path + if sys.platform == "win32": + # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 + sys.modules["pathlib2"].sys.getfilesystemencoding = lambda: "utf-8" + __all__ = ("Path",) From cb19362425798dec1a234f08260a473e896e2af7 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 17:05:07 +0000 Subject: [PATCH 10/18] use utf-8 for activation scripts Signed-off-by: Bernat Gabor --- .../interpreters/discovery/builtin.py | 37 +++++++++---------- tests/unit/activation/conftest.py | 1 + 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py index dfccb5437..e65d03758 100644 --- a/src/virtualenv/interpreters/discovery/builtin.py +++ b/src/virtualenv/interpreters/discovery/builtin.py @@ -4,8 +4,9 @@ import os import sys +import six + from virtualenv.info import IS_WIN -from virtualenv.util import Path from .discover import Discover from .py_info import CURRENT, PythonInfo @@ -63,7 +64,9 @@ def propose_interpreters(spec): yield interpreter, True paths = get_paths() - for path in paths: # find on path, the path order matters (as the candidates are less easy to control by end user) + # find on path, the path order matters (as the candidates are less easy to control by end user) + for pos, path in enumerate(paths): + logging.debug(LazyPathDump(pos, path)) for candidate, match in possible_specs(spec): found = check_path(candidate, path) if found is not None: @@ -84,31 +87,25 @@ def get_paths(): paths = [] else: paths = [p for p in path.split(os.pathsep) if os.path.exists(p)] - logging.debug(LazyPathDump(paths)) return paths class LazyPathDump(object): - def __init__(self, paths): - self.paths = paths + def __init__(self, pos, path): + self.pos = pos + self.path = path def __str__(self): - content = "PATH =>{}".format(os.linesep) - for i, p in enumerate(self.paths): - files = [] - for file in Path(p).iterdir(): - try: - if file.is_dir(): - continue - except OSError: - pass - files.append(file.name) - content += str(i) + content = "discover from PATH[{}]:{} with =>".format(self.pos, self.path) + for file in os.listdir(six.ensure_text(self.path)): + try: + file_path = os.path.join(self.path, file) + if os.path.isdir(file_path) or not os.access(file_path, os.X_OK): + continue + except OSError: + pass content += " " - content += str(p) - content += " with " - content += " ".join(files) - content += os.linesep + content += file return content diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 1384cf501..6f1579abc 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -69,6 +69,7 @@ def env(self, tmp_path): env = os.environ.copy() # add the current python executable folder to the path so we already have another python on the path # also keep the path so the shells (fish, bash, etc can be discovered) + env[str("PYTHONIOENCODING")] = "utf-8" env[str("PATH")] = os.pathsep.join([dirname(sys.executable)] + env.get(str("PATH"), str("")).split(os.pathsep)) # clear up some environment variables so they don't affect the tests for key in [k for k in env.keys() if k.startswith("_OLD") or k.startswith("VIRTUALENV_")]: From 208aa039fbb0115fd22c0ee6517d780db556177d Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 20:42:34 +0000 Subject: [PATCH 11/18] fix --- src/virtualenv/activation/via_template.py | 3 ++- .../interpreters/create/cpython/site.py | 21 ++++++++++--------- src/virtualenv/pyenv_cfg.py | 2 +- src/virtualenv/util/__init__.py | 8 +++---- tests/unit/activation/conftest.py | 20 +++++++++--------- .../unit/activation/test_python_activator.py | 1 + 6 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index e7b3aadf4..1a07ade5b 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,4 +35,5 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - (to_folder / template).write_text(text, encoding="utf-8") + with open(six.ensure_text(str(to_folder / template)), 'wb') as file_handler: + file_handler.write(text.encode('utf-8')) diff --git a/src/virtualenv/interpreters/create/cpython/site.py b/src/virtualenv/interpreters/create/cpython/site.py index 02d892495..8719220e4 100644 --- a/src/virtualenv/interpreters/create/cpython/site.py +++ b/src/virtualenv/interpreters/create/cpython/site.py @@ -9,9 +9,9 @@ def main(): """Patch what needed, and invoke the original site.py""" config = read_pyvenv() - sys.real_prefix = sys.base_prefix = config["base-prefix"] - sys.base_exec_prefix = config["base-exec-prefix"] - global_site_package_enabled = config.get("include-system-site-packages", False) == "true" + sys.real_prefix = sys.base_prefix = config[u"base-prefix"] + sys.base_exec_prefix = config[u"base-exec-prefix"] + global_site_package_enabled = config.get(u"include-system-site-packages", False) == u"true" rewrite_standard_library_sys_path() disable_user_site_package() load_host_site() @@ -38,14 +38,14 @@ def load_host_site(): def read_pyvenv(): """read pyvenv.cfg""" - os_sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version - config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep) - with open(config_file) as file_handler: - lines = file_handler.readlines() + os_sep = u"\\" if sys.platform == "win32" else u"/" # no os module here yet - poor mans version + config_file = u"{}{}pyvenv.cfg".format(sys.prefix.decode(sys.getfilesystemencoding()), os_sep) + with open(config_file, 'rb') as file_handler: + lines = file_handler.read().decode('utf-8').splitlines() config = {} for line in lines: try: - split_at = line.index("=") + split_at = line.index(u"=") except ValueError: continue # ignore bad/empty lines else: @@ -55,8 +55,9 @@ def read_pyvenv(): def rewrite_standard_library_sys_path(): """Once this site file is loaded the standard library paths have already been set, fix them up""" - sep = "\\" if sys.platform == "win32" else "/" - exe_dir = sys.executable[: sys.executable.rfind(sep)] + sep = u"\\" if sys.platform == "win32" else u"/" + exe = sys.executable.decode(sys.getfilesystemencoding()) + exe_dir = exe[: exe.rfind(sep)] for at, value in enumerate(sys.path): # replace old sys prefix path starts with new if value == exe_dir: diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/pyenv_cfg.py index 14860d562..05daa5a3e 100644 --- a/src/virtualenv/pyenv_cfg.py +++ b/src/virtualenv/pyenv_cfg.py @@ -30,7 +30,7 @@ def _read_values(path): return content def write(self): - with open(str(self.path), "wb") as file_handler: + with open(six.ensure_text(str(self.path)), "wb") as file_handler: logging.debug("write %s", six.ensure_text(str(self.path))) for key, value in self.content.items(): line = "{} = {}".format(key, value) diff --git a/src/virtualenv/util/__init__.py b/src/virtualenv/util/__init__.py index a2cb48bc8..7241c1cba 100644 --- a/src/virtualenv/util/__init__.py +++ b/src/virtualenv/util/__init__.py @@ -15,7 +15,7 @@ def ensure_dir(path): if not path.exists(): logging.debug("created %s", six.ensure_text(str(path))) - makedirs(str(path)) + makedirs(six.ensure_text(str(path))) HAS_SYMLINK = hasattr(os, "symlink") @@ -32,9 +32,9 @@ def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): if not dst.is_symlink(): # can't link to itself! if relative_symlinks_ok: assert src.parent == dst.parent - os.symlink(src.name, str(dst)) + os.symlink(six.ensure_text(src.name), six.ensure_text(str(dst))) else: - os.symlink(str(src), str(dst)) + os.symlink(six.ensure_text(str(src)), six.ensure_text(str(dst))) except OSError as exception: logging.warning( "symlink failed %r, for %s to %s, will try copy", @@ -45,7 +45,7 @@ def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): do_copy = True if do_copy: copier = shutil.copy2 if src.is_file() else shutil.copytree - copier(str(src), str(dst)) + copier(six.ensure_text(str(src)), six.ensure_text(str(dst))) logging.debug("%s %s to %s", "copy" if do_copy else "symlink", six.ensure_text(str(src)), six.ensure_text(str(dst))) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 6f1579abc..1c17401db 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -45,10 +45,10 @@ def get_version(self, raise_on_fail): def __call__(self, monkeypatch, tmp_path): activate_script = self._creator.bin_dir / self.activate_script test_script = self._generate_test_script(activate_script, tmp_path) - monkeypatch.chdir(tmp_path) + monkeypatch.chdir(six.ensure_text(str(tmp_path))) monkeypatch.delenv(str("VIRTUAL_ENV"), raising=False) - invoke, env = self._invoke_script + [str(test_script)], self.env(tmp_path) + invoke, env = self._invoke_script + [six.ensure_text(str(test_script))], self.env(tmp_path) try: _raw = subprocess.check_output(invoke, stderr=subprocess.STDOUT, env=env) @@ -69,7 +69,7 @@ def env(self, tmp_path): env = os.environ.copy() # add the current python executable folder to the path so we already have another python on the path # also keep the path so the shells (fish, bash, etc can be discovered) - env[str("PYTHONIOENCODING")] = "utf-8" + env[str("PYTHONIOENCODING")] = str("utf-8") env[str("PATH")] = os.pathsep.join([dirname(sys.executable)] + env.get(str("PATH"), str("")).split(os.pathsep)) # clear up some environment variables so they don't affect the tests for key in [k for k in env.keys() if k.startswith("_OLD") or k.startswith("VIRTUALENV_")]: @@ -80,7 +80,8 @@ def _generate_test_script(self, activate_script, tmp_path): commands = self._get_test_lines(activate_script) script = os.linesep.join(commands) test_script = tmp_path / "script.{}".format(self.extension) - test_script.write_text(script, encoding=self.script_encoding) + with open(six.ensure_text(str(test_script)), 'wb') as file_handler: + file_handler.write(script.encode(self.script_encoding)) return test_script def _get_test_lines(self, activate_script): @@ -174,10 +175,11 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") def activation_python(tmp_path_factory, special_char_name): - dest = tmp_path_factory.mktemp(six.ensure_str("env-{}-v".format(special_char_name))) - session = run_via_cli(["--seed", "none", str(dest), "--prompt", special_char_name]) + dest = tmp_path_factory.mktemp('activation-tester-env') / six.ensure_text("env-{}-v".format(special_char_name)) + session = run_via_cli(["--seed", "none", six.ensure_text(str(dest)), "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" - pydoc_test.write_text('"""This is pydoc_test.py"""') + with open(six.ensure_text(str(pydoc_test)), 'wb') as file_handler: + file_handler.write('"""This is pydoc_test.py"""'.encode('utf-8')) return session @@ -190,8 +192,6 @@ def _tester(tester_class): version = tester.get_version(raise_on_fail=is_inside_ci) if not isinstance(version, six.string_types): pytest.skip(msg=six.text_type(version)) - folder = tmp_path / "test-{}-env".format(special_char_name) - folder.mkdir() - return tester(monkeypatch, folder) + return tester(monkeypatch, tmp_path) return _tester diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index 4070c45f3..9928f7fe3 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -23,6 +23,7 @@ def __init__(self, session): def env(self, tmp_path): env = os.environ.copy() + env[str("PYTHONIOENCODING")] = str("utf-8") for key in {"VIRTUAL_ENV", "PYTHONPATH"}: env.pop(str(key), None) env[str("PATH")] = os.pathsep.join([str(tmp_path), str(tmp_path / "other")]) From 316bb0d7a2ab592e2baadddd79dd03f597a39036 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 20:53:21 +0000 Subject: [PATCH 12/18] more fix Signed-off-by: Bernat Gabor --- src/virtualenv/activation/via_template.py | 4 ++-- .../interpreters/create/cpython/site.py | 21 +++++++++---------- src/virtualenv/interpreters/create/venv.py | 2 +- .../interpreters/discovery/builtin.py | 7 ++++--- tests/unit/activation/conftest.py | 8 +++---- .../unit/interpreters/create/test_creator.py | 12 ++++++++++- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 1a07ade5b..38ccc5660 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,5 +35,5 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - with open(six.ensure_text(str(to_folder / template)), 'wb') as file_handler: - file_handler.write(text.encode('utf-8')) + with open(six.ensure_text(str(to_folder / template)), "wb") as file_handler: + file_handler.write(text.encode("utf-8")) diff --git a/src/virtualenv/interpreters/create/cpython/site.py b/src/virtualenv/interpreters/create/cpython/site.py index 8719220e4..02d892495 100644 --- a/src/virtualenv/interpreters/create/cpython/site.py +++ b/src/virtualenv/interpreters/create/cpython/site.py @@ -9,9 +9,9 @@ def main(): """Patch what needed, and invoke the original site.py""" config = read_pyvenv() - sys.real_prefix = sys.base_prefix = config[u"base-prefix"] - sys.base_exec_prefix = config[u"base-exec-prefix"] - global_site_package_enabled = config.get(u"include-system-site-packages", False) == u"true" + sys.real_prefix = sys.base_prefix = config["base-prefix"] + sys.base_exec_prefix = config["base-exec-prefix"] + global_site_package_enabled = config.get("include-system-site-packages", False) == "true" rewrite_standard_library_sys_path() disable_user_site_package() load_host_site() @@ -38,14 +38,14 @@ def load_host_site(): def read_pyvenv(): """read pyvenv.cfg""" - os_sep = u"\\" if sys.platform == "win32" else u"/" # no os module here yet - poor mans version - config_file = u"{}{}pyvenv.cfg".format(sys.prefix.decode(sys.getfilesystemencoding()), os_sep) - with open(config_file, 'rb') as file_handler: - lines = file_handler.read().decode('utf-8').splitlines() + os_sep = "\\" if sys.platform == "win32" else "/" # no os module here yet - poor mans version + config_file = "{}{}pyvenv.cfg".format(sys.prefix, os_sep) + with open(config_file) as file_handler: + lines = file_handler.readlines() config = {} for line in lines: try: - split_at = line.index(u"=") + split_at = line.index("=") except ValueError: continue # ignore bad/empty lines else: @@ -55,9 +55,8 @@ def read_pyvenv(): def rewrite_standard_library_sys_path(): """Once this site file is loaded the standard library paths have already been set, fix them up""" - sep = u"\\" if sys.platform == "win32" else u"/" - exe = sys.executable.decode(sys.getfilesystemencoding()) - exe_dir = exe[: exe.rfind(sep)] + sep = "\\" if sys.platform == "win32" else "/" + exe_dir = sys.executable[: sys.executable.rfind(sep)] for at, value in enumerate(sys.path): # replace old sys prefix path starts with new if value == exe_dir: diff --git a/src/virtualenv/interpreters/create/venv.py b/src/virtualenv/interpreters/create/venv.py index 6c71d94fb..e80a45ecb 100644 --- a/src/virtualenv/interpreters/create/venv.py +++ b/src/virtualenv/interpreters/create/venv.py @@ -47,7 +47,7 @@ def create_via_sub_process(self): raise ProcessCallFailed(code, out, err, cmd) def get_host_create_cmd(self): - cmd = [str(self.interpreter.system_executable), "-m", "venv", "--without-pip"] + cmd = [self.interpreter.system_executable, "-m", "venv", "--without-pip"] if self.system_site_package: cmd.append("--system-site-packages") cmd.append("--symlinks" if self.symlinks else "--copies") diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py index e65d03758..166066d4c 100644 --- a/src/virtualenv/interpreters/discovery/builtin.py +++ b/src/virtualenv/interpreters/discovery/builtin.py @@ -66,6 +66,7 @@ def propose_interpreters(spec): paths = get_paths() # find on path, the path order matters (as the candidates are less easy to control by end user) for pos, path in enumerate(paths): + path = six.ensure_text(path) logging.debug(LazyPathDump(pos, path)) for candidate, match in possible_specs(spec): found = check_path(candidate, path) @@ -97,15 +98,15 @@ def __init__(self, pos, path): def __str__(self): content = "discover from PATH[{}]:{} with =>".format(self.pos, self.path) - for file in os.listdir(six.ensure_text(self.path)): + for file_name in os.listdir(self.path): try: - file_path = os.path.join(self.path, file) + file_path = os.path.join(self.path, file_name) if os.path.isdir(file_path) or not os.access(file_path, os.X_OK): continue except OSError: pass content += " " - content += file + content += file_name return content diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 1c17401db..5369cfa48 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -80,7 +80,7 @@ def _generate_test_script(self, activate_script, tmp_path): commands = self._get_test_lines(activate_script) script = os.linesep.join(commands) test_script = tmp_path / "script.{}".format(self.extension) - with open(six.ensure_text(str(test_script)), 'wb') as file_handler: + with open(six.ensure_text(str(test_script)), "wb") as file_handler: file_handler.write(script.encode(self.script_encoding)) return test_script @@ -175,11 +175,11 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") def activation_python(tmp_path_factory, special_char_name): - dest = tmp_path_factory.mktemp('activation-tester-env') / six.ensure_text("env-{}-v".format(special_char_name)) + dest = tmp_path_factory.mktemp("activation-tester-env") / six.ensure_text("env-{}-v".format(special_char_name)) session = run_via_cli(["--seed", "none", six.ensure_text(str(dest)), "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" - with open(six.ensure_text(str(pydoc_test)), 'wb') as file_handler: - file_handler.write('"""This is pydoc_test.py"""'.encode('utf-8')) + with open(six.ensure_text(str(pydoc_test)), "wb") as file_handler: + file_handler.write(b'"""This is pydoc_test.py"""') return session diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 7aef9001a..374d5774c 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -218,7 +218,17 @@ def cross_python(is_inside_ci): def test_cross_major(cross_python, coverage_env, tmp_path): - cmd = ["-v", "-v", "-p", str(cross_python.executable), str(tmp_path), "--seeder", "none", "--activators", ""] + cmd = [ + "-v", + "-v", + "-p", + six.ensure_text(cross_python.executable), + six.ensure_text(str(tmp_path)), + "--seeder", + "none", + "--activators", + "", + ] result = run_via_cli(cmd) coverage_env() env = PythonInfo.from_exe(str(result.creator.exe)) From 1e782d97e270a37b0470a36811bef3afc783dd3f Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 30 Dec 2019 21:53:50 +0000 Subject: [PATCH 13/18] fix Signed-off-by: Bernat Gabor --- src/virtualenv/activation/via_template.py | 3 +-- src/virtualenv/interpreters/create/creator.py | 7 +++++++ src/virtualenv/util/path.py | 4 +++- tests/unit/activation/conftest.py | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 38ccc5660..34b06fa8f 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,5 +35,4 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - with open(six.ensure_text(str(to_folder / template)), "wb") as file_handler: - file_handler.write(text.encode("utf-8")) + (to_folder / template).write_text(text) diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index c8aa3d83f..a89af4208 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -55,6 +55,13 @@ def validate_dest_dir(raw_value): "destination {!r} must not contain the path separator ({}) as this would break " "the activation scripts".format(raw_value, os.pathsep) ) + if six.PY2 and sys.platform == "win32": + path_converted = raw_value.encode("mbcs") + if path_converted != raw_value: + raise ArgumentTypeError( + "mbcs (path encoder for CPython2.7 on Windows) does not support one or more characters" + ) + value = Path(raw_value) if value.exists() and value.is_file(): raise ArgumentTypeError("the destination {} already exists and is a file".format(value)) diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index 3c6be2a5f..ad2a8ad3c 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -31,6 +31,8 @@ def write_text(self, data, encoding=None, errors=None): if sys.platform == "win32": # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 - sys.modules["pathlib2"].sys.getfilesystemencoding = lambda: "utf-8" + class Path(object): + pass + __all__ = ("Path",) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 5369cfa48..e13b0a1bb 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -175,8 +175,8 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") def activation_python(tmp_path_factory, special_char_name): - dest = tmp_path_factory.mktemp("activation-tester-env") / six.ensure_text("env-{}-v".format(special_char_name)) - session = run_via_cli(["--seed", "none", six.ensure_text(str(dest)), "--prompt", special_char_name]) + dest = os.path.join(six.ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), six.ensure_text("env-{}-v".format(special_char_name))) + session = run_via_cli(["--seed", "none", dest, "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" with open(six.ensure_text(str(pydoc_test)), "wb") as file_handler: file_handler.write(b'"""This is pydoc_test.py"""') From 8d010f3c26485eba52ff822443cecf90e680012a Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 31 Dec 2019 10:52:55 +0000 Subject: [PATCH 14/18] windows path py2.7 --- src/virtualenv/interpreters/create/creator.py | 2 +- src/virtualenv/util/path.py | 73 ++++++++++++++++++- tests/unit/activation/conftest.py | 5 +- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index a89af4208..e96549bdc 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -73,7 +73,7 @@ def validate_dest_dir(raw_value): value = dest while dest: if dest.exists(): - if os.access(str(dest), os.W_OK): + if os.access(six.ensure_text(str(dest)), os.W_OK): break else: non_write_able(dest, value) diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index ad2a8ad3c..7073682a4 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -1,4 +1,5 @@ import sys +from contextlib import contextmanager import six @@ -31,8 +32,78 @@ def write_text(self, data, encoding=None, errors=None): if sys.platform == "win32": # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 + import os + class Path(object): - pass + def __init__(self, path): + self.path = six.ensure_text(path) + + def __div__(self, other): + return Path(os.path.join(self.path, other.path if isinstance(other, Path) else six.ensure_text(other))) + + def exists(self): + return os.path.exists(self.path) + + def absolute(self): + return Path(os.path.abspath(self.path)) + + @property + def parent(self): + return Path(os.path.abspath(os.path.join(self.path, os.path.pardir))) + + def resolve(self): + return Path(os.path.realpath(self.path)) + + @property + def name(self): + return os.path.basename(self.path) + + @property + def parts(self): + return self.path.split(os.sep) + + def is_file(self): + return os.path.isfile(self.path) + + def is_dir(self): + return os.path.isdir(self.path) + + def __repr__(self): + return "Path({})".format(self.path) + + def __str__(self): + return self.path.decode("utf-8") + + def mkdir(self, parents=True, exist_ok=True): + if not self.exists() and exist_ok: + os.makedirs(self.path) + + def read_text(self, encoding="utf-8"): + with open(self.path, "rb") as file_handler: + return file_handler.read().decode(encoding) + + def write_text(self, text, encoding="utf-8"): + with open(self.path, "wb") as file_handler: + file_handler.write(text.encode(encoding)) + + def iterdir(self): + for p in os.listdir(self.path): + yield Path(os.path.join(self.path, p)) + + @property + def suffix(self): + _, ext = os.path.splitext(self.name) + return ext + + @property + def stem(self): + base, _ = os.path.splitext(self.name) + return base + + @contextmanager + def open(self): + with open(self.path) as file_handler: + yield file_handler __all__ = ("Path",) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index e13b0a1bb..dafdfb8f1 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -175,7 +175,10 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session") def activation_python(tmp_path_factory, special_char_name): - dest = os.path.join(six.ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), six.ensure_text("env-{}-v".format(special_char_name))) + dest = os.path.join( + six.ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), + six.ensure_text("env-{}-v".format(special_char_name)), + ) session = run_via_cli(["--seed", "none", dest, "--prompt", special_char_name]) pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" with open(six.ensure_text(str(pydoc_test)), "wb") as file_handler: From 378b8a018594456d840b58dc4d25124a2474cf02 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 2 Jan 2020 14:19:43 +0000 Subject: [PATCH 15/18] fixes for Python 2 and unicode on Windows --- .../activation/python/activate_this.py | 15 +- src/virtualenv/activation/via_template.py | 2 +- .../interpreters/create/cpython/cpython2.py | 15 ++ src/virtualenv/interpreters/create/creator.py | 71 ++++---- src/virtualenv/interpreters/create/debug.py | 30 +++- .../interpreters/discovery/py_info.py | 7 +- src/virtualenv/seed/embed/pip_invoke.py | 5 +- src/virtualenv/seed/embed/wheels/acquire.py | 2 +- src/virtualenv/util/__init__.py | 5 +- src/virtualenv/util/path.py | 78 ++++++--- src/virtualenv/util/subprocess/__init__.py | 15 ++ .../util/subprocess/win_subprocess.py | 153 ++++++++++++++++++ tests/conftest.py | 20 ++- tests/unit/activation/conftest.py | 32 ++-- .../unit/activation/test_python_activator.py | 28 ++-- tests/unit/config/test___main__.py | 7 +- .../test_boostrap_link_via_app_data.py | 16 +- tests/unit/interpreters/create/conftest.py | 16 +- .../unit/interpreters/create/test_creator.py | 35 ++-- tox.ini | 2 +- 20 files changed, 418 insertions(+), 136 deletions(-) create mode 100644 src/virtualenv/util/subprocess/__init__.py create mode 100644 src/virtualenv/util/subprocess/win_subprocess.py diff --git a/src/virtualenv/activation/python/activate_this.py b/src/virtualenv/activation/python/activate_this.py index dc1e977ab..3311fe813 100644 --- a/src/virtualenv/activation/python/activate_this.py +++ b/src/virtualenv/activation/python/activate_this.py @@ -15,14 +15,21 @@ except NameError: raise AssertionError("You must use exec(open(this_file).read(), {'__file__': this_file}))") + +def set_env(key, value, encoding): + if sys.version_info[0] == 2: + value = value.encode(encoding) + os.environ[key] = value + + # prepend bin to PATH (this file is inside the bin directory) bin_dir = os.path.dirname(os.path.abspath(__file__)) -os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep)) +set_env("PATH", os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep)), sys.getfilesystemencoding()) base = os.path.dirname(bin_dir) # virtual env is right above bin directory -os.environ["VIRTUAL_ENV"] = base +set_env("VIRTUAL_ENV", base, sys.getfilesystemencoding()) # add the virtual environments site-packages to the host python import mechanism prev = set(sys.path) @@ -35,8 +42,10 @@ for site_package in json.loads(site_packages): if sys.version_info[0] == 2: - site_package = site_package.encode('utf-8') + site_package = site_package.encode('utf-8').decode(sys.getfilesystemencoding()) path = os.path.realpath(os.path.join(os.path.dirname(__file__), site_package)) + if sys.version_info[0] == 2: + path = path.encode(sys.getfilesystemencoding()) site.addsitedir(path) # fmt: on diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 34b06fa8f..e7b3aadf4 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -35,4 +35,4 @@ def _generate(self, replacements, templates, to_folder): text = pkgutil.get_data(self.__module__, str(template)).decode("utf-8") for start, end in replacements.items(): text = text.replace(start, end) - (to_folder / template).write_text(text) + (to_folder / template).write_text(text, encoding="utf-8") diff --git a/src/virtualenv/interpreters/create/cpython/cpython2.py b/src/virtualenv/interpreters/create/cpython/cpython2.py index 7c7915882..bacefeeb9 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython2.py +++ b/src/virtualenv/interpreters/create/cpython/cpython2.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, unicode_literals import abc +from argparse import ArgumentTypeError import six @@ -67,3 +68,17 @@ def add_folder(self, folder): class CPython2Windows(CPython2, CPythonWindows): """CPython 2 on Windows""" + + @classmethod + def validate_dest_dir(cls, raw_value): + # the python prefix discovery mechanism on Windows python 2 uses mbcs - so anything that's not encode-able + # needs to be refused + path_converted = raw_value.encode("mbcs", errors="ignore").decode("mbcs") + if path_converted != raw_value: + refused = set(raw_value) - { + c for c, i in ((char, char.encode("mbcs")) for char in raw_value) if c == "?" or i != "?" + } + raise ArgumentTypeError( + "mbcs (path encoder for CPython2.7 on Windows) does not support characters %r".format(refused) + ) + return super(CPython2Windows, cls).validate_dest_dir(raw_value) diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index e96549bdc..69aec645a 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -48,50 +48,45 @@ def add_parser_arguments(cls, parser, interpreter): help="Give the virtual environment access to the system site-packages dir.", ) - def validate_dest_dir(raw_value): - """No path separator in the path and must be write-able""" - if os.pathsep in raw_value: - raise ArgumentTypeError( - "destination {!r} must not contain the path separator ({}) as this would break " - "the activation scripts".format(raw_value, os.pathsep) - ) - if six.PY2 and sys.platform == "win32": - path_converted = raw_value.encode("mbcs") - if path_converted != raw_value: - raise ArgumentTypeError( - "mbcs (path encoder for CPython2.7 on Windows) does not support one or more characters" - ) - - value = Path(raw_value) - if value.exists() and value.is_file(): - raise ArgumentTypeError("the destination {} already exists and is a file".format(value)) - if (3, 3) <= sys.version_info <= (3, 6): - # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation - dest = Path(os.path.realpath(raw_value)) - else: - dest = value.resolve() - value = dest - while dest: - if dest.exists(): - if os.access(six.ensure_text(str(dest)), os.W_OK): - break - else: - non_write_able(dest, value) - base, _ = dest.parent, dest.name - if base == dest: - non_write_able(dest, value) # pragma: no cover - dest = base - return str(value) + parser.add_argument( + "dest_dir", help="directory to create virtualenv at", type=cls.validate_dest_dir, default="env", nargs="?", + ) + @classmethod + def validate_dest_dir(cls, raw_value): def non_write_able(dest, value): common = Path(*os.path.commonprefix([value.parts, dest.parts])) raise ArgumentTypeError( "the destination {} is not write-able at {}".format(dest.relative_to(common), common) ) - parser.add_argument( - "dest_dir", help="directory to create virtualenv at", type=validate_dest_dir, default="env", nargs="?", - ) + """No path separator in the path and must be write-able""" + if os.pathsep in raw_value: + raise ArgumentTypeError( + "destination {!r} must not contain the path separator ({}) as this would break " + "the activation scripts".format(raw_value, os.pathsep) + ) + + value = Path(raw_value) + if value.exists() and value.is_file(): + raise ArgumentTypeError("the destination {} already exists and is a file".format(value)) + if (3, 3) <= sys.version_info <= (3, 6): + # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation + dest = Path(os.path.realpath(raw_value)) + else: + dest = value.resolve() + value = dest + while dest: + if dest.exists(): + if os.access(six.ensure_text(str(dest)), os.W_OK): + break + else: + non_write_able(dest, value) + base, _ = dest.parent, dest.name + if base == dest: + non_write_able(dest, value) # pragma: no cover + dest = base + return str(value) def run(self): if self.dest_dir.exists() and self.clear: @@ -151,7 +146,7 @@ def debug_script(self): def get_env_debug_info(env_exe, debug_script): - cmd = [str(env_exe), str(debug_script)] + cmd = [six.ensure_text(str(env_exe)), six.ensure_text(str(debug_script))] logging.debug(" ".join(six.ensure_text(i) for i in cmd)) env = os.environ.copy() env.pop("PYTHONPATH", None) diff --git a/src/virtualenv/interpreters/create/debug.py b/src/virtualenv/interpreters/create/debug.py index cb4da8e8e..37f1a4507 100644 --- a/src/virtualenv/interpreters/create/debug.py +++ b/src/virtualenv/interpreters/create/debug.py @@ -2,6 +2,20 @@ import sys # built-in +def encode_path(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode(sys.getfilesystemencoding()) + if isinstance(value, type): + return repr(value) + return value + + +def encode_list_path(value): + return [encode_path(i) for i in value] + + def run(): """print debug data about the virtual environment""" try: @@ -11,7 +25,7 @@ def run(): # noinspection PyPep8Naming OrderedDict = dict # pragma: no cover result = OrderedDict([("sys", OrderedDict())]) - for key in ( + path_keys = ( "executable", "_base_executable", "prefix", @@ -21,13 +35,15 @@ def run(): "base_exec_prefix", "path", "meta_path", - "version", - ): + ) + for key in path_keys: value = getattr(sys, key, None) - if key == "meta_path" and value is not None: - value = [repr(i) for i in value] + if isinstance(value, list): + value = encode_list_path(value) + else: + value = encode_path(value) result["sys"][key] = value - + result["version"] = sys.version import os # landmark result["os"] = os.__file__ @@ -45,7 +61,7 @@ def run(): result["json"] = repr(json) print(json.dumps(result, indent=2)) - except ImportError as exception: # pragma: no cover + except (ImportError, ValueError, TypeError) as exception: # pragma: no cover result["json"] = repr(exception) # pragma: no cover print(repr(result)) # pragma: no cover raise SystemExit(1) # pragma: no cover diff --git a/src/virtualenv/interpreters/discovery/py_info.py b/src/virtualenv/interpreters/discovery/py_info.py index 2af239412..13e9b795d 100644 --- a/src/virtualenv/interpreters/discovery/py_info.py +++ b/src/virtualenv/interpreters/discovery/py_info.py @@ -10,12 +10,9 @@ import logging import os import platform -import subprocess import sys from collections import OrderedDict, namedtuple -IS_WIN = sys.platform == "win32" - VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) @@ -205,12 +202,14 @@ def from_exe(cls, exe, raise_on_error=True): @classmethod def _load_for_exe(cls, exe): + from virtualenv.util.subprocess import subprocess, Popen + path = "{}.py".format(os.path.splitext(__file__)[0]) cmd = [exe, path] # noinspection DuplicatedCode # this is duplicated here because this file is executed on its own, so cannot be refactored otherwise try: - process = subprocess.Popen( + process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) out, err = process.communicate() diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 4de49f3d6..d65e89a77 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -1,10 +1,10 @@ from __future__ import absolute_import, unicode_literals import os -import subprocess from virtualenv.seed.embed.base_embed import BaseEmbed from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel +from virtualenv.util.subprocess import Popen class PipInvoke(BaseEmbed): @@ -39,4 +39,5 @@ def run(self, creator): } ) - subprocess.call(cmd, env=env) + process = Popen(cmd, env=env) + process.communicate() diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index fcfb26357..6460b31a5 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -1,11 +1,11 @@ """Bootstrap""" from __future__ import absolute_import, unicode_literals -import subprocess from collections import defaultdict from shutil import copy2 from virtualenv.util import Path +from virtualenv.util.subprocess import subprocess from . import BUNDLE_SUPPORT, MAX diff --git a/src/virtualenv/util/__init__.py b/src/virtualenv/util/__init__.py index 7241c1cba..9e7d13bad 100644 --- a/src/virtualenv/util/__init__.py +++ b/src/virtualenv/util/__init__.py @@ -3,12 +3,13 @@ import logging import os import shutil -import subprocess from functools import partial from os import makedirs import six +from virtualenv.util.subprocess import Popen, subprocess + from .path import Path @@ -51,7 +52,7 @@ def symlink_or_copy(do_copy, src, dst, relative_symlinks_ok=False): def run_cmd(cmd): try: - process = subprocess.Popen( + process = Popen( cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) out, err = process.communicate() # input disabled diff --git a/src/virtualenv/util/path.py b/src/virtualenv/util/path.py index 7073682a4..5ef51867d 100644 --- a/src/virtualenv/util/path.py +++ b/src/virtualenv/util/path.py @@ -7,6 +7,7 @@ from pathlib import Path if sys.version_info[0:2] == (3, 4): + # no read/write text on python3.4 BuiltinPath = Path class Path(type(BuiltinPath())): @@ -28,67 +29,76 @@ def write_text(self, data, encoding=None, errors=None): else: - from pathlib2 import Path - if sys.platform == "win32": # workaround for https://github.com/mcmtroffaes/pathlib2/issues/56 import os class Path(object): def __init__(self, path): - self.path = six.ensure_text(path) + self._path = path._path if isinstance(path, Path) else six.ensure_text(path) + + def __repr__(self): + return six.ensure_str(u"Path({})".format(self._path)) + + def __str__(self): + return six.ensure_str(self._path) def __div__(self, other): - return Path(os.path.join(self.path, other.path if isinstance(other, Path) else six.ensure_text(other))) + return Path( + os.path.join(self._path, other._path if isinstance(other, Path) else six.ensure_text(other)) + ) + + def __eq__(self, other): + return self._path == (other._path if isinstance(other, Path) else None) + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self._path) def exists(self): - return os.path.exists(self.path) + return os.path.exists(self._path) def absolute(self): - return Path(os.path.abspath(self.path)) + return Path(os.path.abspath(self._path)) @property def parent(self): - return Path(os.path.abspath(os.path.join(self.path, os.path.pardir))) + return Path(os.path.abspath(os.path.join(self._path, os.path.pardir))) def resolve(self): - return Path(os.path.realpath(self.path)) + return Path(os.path.realpath(self._path)) @property def name(self): - return os.path.basename(self.path) + return os.path.basename(self._path) @property def parts(self): - return self.path.split(os.sep) + return self._path.split(os.sep) def is_file(self): - return os.path.isfile(self.path) + return os.path.isfile(self._path) def is_dir(self): - return os.path.isdir(self.path) - - def __repr__(self): - return "Path({})".format(self.path) - - def __str__(self): - return self.path.decode("utf-8") + return os.path.isdir(self._path) def mkdir(self, parents=True, exist_ok=True): if not self.exists() and exist_ok: - os.makedirs(self.path) + os.makedirs(self._path) def read_text(self, encoding="utf-8"): - with open(self.path, "rb") as file_handler: + with open(self._path, "rb") as file_handler: return file_handler.read().decode(encoding) def write_text(self, text, encoding="utf-8"): - with open(self.path, "wb") as file_handler: + with open(self._path, "wb") as file_handler: file_handler.write(text.encode(encoding)) def iterdir(self): - for p in os.listdir(self.path): - yield Path(os.path.join(self.path, p)) + for p in os.listdir(self._path): + yield Path(os.path.join(self._path, p)) @property def suffix(self): @@ -101,9 +111,27 @@ def stem(self): return base @contextmanager - def open(self): - with open(self.path) as file_handler: + def open(self, mode="r"): + with open(self._path, mode) as file_handler: yield file_handler + @property + def parents(self): + result = [] + parts = self.parts + for i in range(len(parts)): + result.append(Path(os.sep.join(parts[0 : i + 1]))) + return result + + def unlink(self): + os.remove(self._path) + + def with_name(self, name): + return self.parent / name + + def is_symlink(self): + return os.path.islink(self._path) + else: + from pathlib2 import Path __all__ = ("Path",) diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py new file mode 100644 index 000000000..a980cae6c --- /dev/null +++ b/src/virtualenv/util/subprocess/__init__.py @@ -0,0 +1,15 @@ +from __future__ import absolute_import, unicode_literals + +import subprocess +import sys + +import six + +if six.PY2 and sys.platform == "win32": + from . import win_subprocess + + Popen = win_subprocess.Popen +else: + Popen = subprocess.Popen + +__all__ = ("subprocess", "Popen") diff --git a/src/virtualenv/util/subprocess/win_subprocess.py b/src/virtualenv/util/subprocess/win_subprocess.py new file mode 100644 index 000000000..e8fdaf00a --- /dev/null +++ b/src/virtualenv/util/subprocess/win_subprocess.py @@ -0,0 +1,153 @@ +# flake8: noqa +# fmt: off +## issue: https://bugs.python.org/issue19264 + +import ctypes +import os +import subprocess +import sys +from ctypes import Structure, WinError, byref, c_char_p, c_void_p, c_wchar, c_wchar_p, sizeof, windll +from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPVOID, LPWSTR, WORD + +import _subprocess + +## +## Types +## + +CREATE_UNICODE_ENVIRONMENT = 0x00000400 +LPCTSTR = c_char_p +LPTSTR = c_wchar_p +LPSECURITY_ATTRIBUTES = c_void_p +LPBYTE = ctypes.POINTER(BYTE) + +class STARTUPINFOW(Structure): + _fields_ = [ + ("cb", DWORD), ("lpReserved", LPWSTR), + ("lpDesktop", LPWSTR), ("lpTitle", LPWSTR), + ("dwX", DWORD), ("dwY", DWORD), + ("dwXSize", DWORD), ("dwYSize", DWORD), + ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), + ("dwFillAtrribute", DWORD), ("dwFlags", DWORD), + ("wShowWindow", WORD), ("cbReserved2", WORD), + ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), + ("hStdOutput", HANDLE), ("hStdError", HANDLE), + ] + +LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW) + + +class PROCESS_INFORMATION(Structure): + _fields_ = [ + ("hProcess", HANDLE), ("hThread", HANDLE), + ("dwProcessId", DWORD), ("dwThreadId", DWORD), + ] + +LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION) + + +class DUMMY_HANDLE(ctypes.c_void_p): + + def __init__(self, *a, **kw): + super(DUMMY_HANDLE, self).__init__(*a, **kw) + self.closed = False + + def Close(self): + if not self.closed: + windll.kernel32.CloseHandle(self) + self.closed = True + + def __int__(self): + return self.value + + +CreateProcessW = windll.kernel32.CreateProcessW +CreateProcessW.argtypes = [ + LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES, + LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR, + LPSTARTUPINFOW, LPPROCESS_INFORMATION, +] +CreateProcessW.restype = BOOL + + +## +## Patched functions/classes +## + +def CreateProcess(executable, args, _p_attr, _t_attr, + inherit_handles, creation_flags, env, cwd, + startup_info): + """Create a process supporting unicode executable and args for win32 + + Python implementation of CreateProcess using CreateProcessW for Win32 + + """ + + si = STARTUPINFOW( + dwFlags=startup_info.dwFlags, + wShowWindow=startup_info.wShowWindow, + cb=sizeof(STARTUPINFOW), + ## XXXvlab: not sure of the casting here to ints. + hStdInput=startup_info.hStdInput if startup_info.hStdInput is None else int(startup_info.hStdInput), + hStdOutput=startup_info.hStdOutput if startup_info.hStdOutput is None else int(startup_info.hStdOutput), + hStdError=startup_info.hStdError if startup_info.hStdError is None else int(startup_info.hStdError), + ) + + wenv = None + if env is not None: + ## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar + env = (unicode("").join([ + unicode("%s=%s\0") % (k, v) + for k, v in env.items()])) + unicode("\0") + wenv = (c_wchar * len(env))() + wenv.value = env + + pi = PROCESS_INFORMATION() + creation_flags |= CREATE_UNICODE_ENVIRONMENT + + if CreateProcessW(executable, args, None, None, + inherit_handles, creation_flags, + wenv, cwd, byref(si), byref(pi)): + return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread), + pi.dwProcessId, pi.dwThreadId) + raise WinError() + + +class Popen(subprocess.Popen): + """This superseeds Popen and corrects a bug in cPython 2.7 implem""" + + def _execute_child(self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + """Code from part of _execute_child from Python 2.7 (9fbb65e) + + There are only 2 little changes concerning the construction of + the the final string in shell mode: we preempt the creation of + the command string when shell is True, because original function + will try to encode unicode args which we want to avoid to be able to + sending it as-is to ``CreateProcess``. + + """ + if startupinfo is None: + startupinfo = subprocess.STARTUPINFO() + if not isinstance(args, subprocess.types.StringTypes): + args = subprocess.list2cmdline(args) + startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = _subprocess.SW_HIDE + comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) + if (_subprocess.GetVersion() >= 0x80000000 or + os.path.basename(comspec).lower() == "command.com"): + w9xpopen = self._find_w9xpopen() + args = unicode('"%s" %s') % (w9xpopen, args) + creationflags |= _subprocess.CREATE_NEW_CONSOLE + + super(Popen, self)._execute_child(args, executable, + preexec_fn, close_fds, cwd, env, universal_newlines, + startupinfo, creationflags, False, to_close, p2cread, + p2cwrite, c2pread, c2pwrite, errread, errwrite) + +_subprocess.CreateProcess = CreateProcess +# fmt: on diff --git a/tests/conftest.py b/tests/conftest.py index d6bd4baf7..0ff2c1f58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import coverage import pytest +import six from virtualenv.interpreters.discovery.py_info import PythonInfo from virtualenv.util import Path @@ -225,4 +226,21 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): - return "πŸš’ Γ¨Ρ€Ρ‚$β™žδΈ­η‰‡" + base = "$ Γ¨Ρ€Ρ‚πŸš’β™žδΈ­η‰‡" + if sys.platform == "win32" and six.PY2: + # PY2 windows uses mbcs as path encoder, so don't try to use what's not encode-able by that + result = "" + for char in base: + encoded = char.encode("mbcs", errors="ignore") + if char == "?" or encoded != six.ensure_str("?"): + result += char + base = result + return base + + +@pytest.fixture() +def special_name_dir(tmp_path, special_char_name): + dest = Path(str(tmp_path)) / special_char_name + yield dest + if six.PY2 and sys.platform == "win32": # pytest python2 windows does not support unicode delete + shutil.rmtree(six.ensure_text(str(dest))) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index dafdfb8f1..ab8c28159 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -3,6 +3,7 @@ import os import pipes import re +import shutil import subprocess import sys from os.path import dirname, normcase, realpath @@ -12,6 +13,7 @@ from virtualenv.run import run_via_cli from virtualenv.util import Path +from virtualenv.util.subprocess import Popen class ActivationTester(object): @@ -30,9 +32,7 @@ def __init__(self, of_class, session, cmd, activate_script, extension): def get_version(self, raise_on_fail): # locally we disable, so that contributors don't need to have everything setup try: - process = subprocess.Popen( - self._version_cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) + process = Popen(self._version_cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if out: return out @@ -51,13 +51,14 @@ def __call__(self, monkeypatch, tmp_path): invoke, env = self._invoke_script + [six.ensure_text(str(test_script))], self.env(tmp_path) try: - _raw = subprocess.check_output(invoke, stderr=subprocess.STDOUT, env=env) - raw = "\n{}".format(_raw.decode("utf-8")) + process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) + _raw, _ = process.communicate() + raw = "\n{}".format(_raw.decode("utf-8")).replace("\r\n", "\n") except subprocess.CalledProcessError as exception: assert not exception.returncode, six.ensure_text(exception.output) return - out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split(os.linesep) + out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().split("\n") self.assert_output(out, raw, tmp_path) return env, activate_script @@ -121,11 +122,18 @@ def python_cmd(self, cmd): return "{} -c {}".format(os.path.basename(sys.executable), self.quote(cmd)) def print_python_exe(self): - return self.python_cmd("import sys; print(sys.executable)") + return self.python_cmd( + "import sys; e = sys.executable;" + "print(e.decode(sys.getfilesystemencoding()) if sys.version_info[0] == 2 else e)" + ) def print_os_env_var(self, var): val = '"{}"'.format(var) - return self.python_cmd("import os; print(os.environ.get({}, None))".format(val)) + return self.python_cmd( + "import os; import sys; v = os.environ.get({}, None);" + "print(v if v is None else " + "(v.decode(sys.getfilesystemencoding()) if sys.version_info[0] == 2 else v))".format(val) + ) def activate_call(self, script): cmd = self.quote(six.ensure_text(str(self.activate_cmd))) @@ -155,7 +163,7 @@ def __init__(self, of_class, session, cmd, activate_script, extension, non_sourc def __call__(self, monkeypatch, tmp_path): env, activate_script = super(RaiseOnNonSourceCall, self).__call__(monkeypatch, tmp_path) - process = subprocess.Popen( + process = Popen( self.non_source_activate(activate_script), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) out, err = process.communicate() @@ -183,11 +191,13 @@ def activation_python(tmp_path_factory, special_char_name): pydoc_test = session.creator.site_packages[0] / "pydoc_test.py" with open(six.ensure_text(str(pydoc_test)), "wb") as file_handler: file_handler.write(b'"""This is pydoc_test.py"""') - return session + yield session + if six.PY2 and sys.platform == "win32": # PY2 windows does not support unicode delete + shutil.rmtree(dest) @pytest.fixture() -def activation_tester(activation_python, monkeypatch, tmp_path, special_char_name, is_inside_ci): +def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci): def _tester(tester_class): tester = tester_class(activation_python) if not tester.of_class.supports(activation_python.creator.interpreter): diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index 9928f7fe3..2a125c767 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -31,8 +31,9 @@ def env(self, tmp_path): def _get_test_lines(self, activate_script): raw = inspect.getsource(self.activate_this_test) - return ["# -*- coding: utf-8 -*-"] + [ - i[12:] for i in raw.replace("__FILENAME__", six.ensure_text(str(activate_script))).splitlines()[2:] + return [ + i[12:] + for i in raw.replace('"__FILENAME__"', repr(six.ensure_text(str(activate_script)))).splitlines()[2:] ] # noinspection PyUnresolvedReferences @@ -41,20 +42,25 @@ def activate_this_test(): import os import sys - print(os.environ.get("VIRTUAL_ENV")) - print(os.environ.get("PATH")) - print(os.pathsep.join(sys.path)) - file_at = r"__FILENAME__" - with open(file_at, "rt") as file_handler: + def print_path(value): + if value is not None and sys.version_info[0] == 2: + value = value.decode(sys.getfilesystemencoding()) + print(value) + + print_path(os.environ.get("VIRTUAL_ENV")) + print_path(os.environ.get("PATH")) + print_path(os.pathsep.join(sys.path)) + file_at = "__FILENAME__" + with open(file_at, "rb") as file_handler: content = file_handler.read() exec(content, {"__file__": file_at}) - print(os.environ.get("VIRTUAL_ENV")) - print(os.environ.get("PATH")) - print(os.pathsep.join(sys.path)) + print_path(os.environ.get("VIRTUAL_ENV")) + print_path(os.environ.get("PATH")) + print_path(os.pathsep.join(sys.path)) import inspect import pydoc_test - print(inspect.getsourcefile(pydoc_test)) + print_path(inspect.getsourcefile(pydoc_test)) def assert_output(self, out, raw, tmp_path): assert out[0] == "None" # start with VIRTUAL_ENV None diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index 3f4663058..91fef2774 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -1,9 +1,12 @@ from __future__ import absolute_import, unicode_literals -import subprocess import sys +from virtualenv.util.subprocess import Popen, subprocess + def test_main(): - out = subprocess.check_output([sys.executable, "-m", "virtualenv", "--help"], universal_newlines=True) + process = Popen([sys.executable, "-m", "virtualenv", "--help"], universal_newlines=True, stdout=subprocess.PIPE) + out, _ = process.communicate() + assert not process.returncode assert out diff --git a/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py b/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py index 9b1ba4e50..3da4e836b 100644 --- a/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py +++ b/tests/unit/interpreters/boostrap/test_boostrap_link_via_app_data.py @@ -1,12 +1,14 @@ from __future__ import absolute_import, unicode_literals import os -import subprocess import sys +import six + from virtualenv.interpreters.discovery.py_info import CURRENT from virtualenv.run import run_via_cli from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT +from virtualenv.util.subprocess import Popen def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env): @@ -44,7 +46,9 @@ def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env): ) ]: assert pip_exe.exists() - subprocess.check_output([str(pip_exe), "--version", "--disable-pip-version-check"]) + process = Popen([six.ensure_text(str(pip_exe)), "--version", "--disable-pip-version-check"]) + _, __ = process.communicate() + assert not process.returncode remove_cmd = [ str(env_exe), @@ -56,7 +60,9 @@ def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env): "-y", "setuptools", ] - assert not subprocess.check_call(remove_cmd) + process = Popen(remove_cmd) + _, __ = process.communicate() + assert not process.returncode assert site_package.exists() files_post_first_uninstall = list(site_package.iterdir()) @@ -70,7 +76,9 @@ def test_base_bootstrap_link_via_app_data(tmp_path, coverage_env): files_post_second_create = list(site_package.iterdir()) assert files_post_first_create == files_post_second_create - assert not subprocess.check_call(remove_cmd + ["pip"]) + process = Popen(remove_cmd + ["pip"]) + _, __ = process.communicate() + assert not process.returncode # pip is greedy here, removing all packages removes the site-package too if site_package.exists(): post_run = list(site_package.iterdir()) diff --git a/tests/unit/interpreters/create/conftest.py b/tests/unit/interpreters/create/conftest.py index 027b6b47b..d673a8a78 100644 --- a/tests/unit/interpreters/create/conftest.py +++ b/tests/unit/interpreters/create/conftest.py @@ -8,12 +8,13 @@ """ from __future__ import absolute_import, unicode_literals -import subprocess import sys import pytest -from virtualenv.interpreters.discovery.py_info import CURRENT, IS_WIN +from virtualenv.info import IS_WIN +from virtualenv.interpreters.discovery.py_info import CURRENT +from virtualenv.util.subprocess import Popen # noinspection PyUnusedLocal @@ -27,7 +28,8 @@ def get_venv(tmp_path_factory): elif CURRENT.version_info.major == 3: root_python = get_root(tmp_path_factory) dest = tmp_path_factory.mktemp("venv") - subprocess.check_call([str(root_python), "-m", "venv", "--without-pip", str(dest)]) + process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) + process.communicate() # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic return CURRENT.find_exe_based_of(inside_folder=str(dest)) @@ -45,11 +47,15 @@ def get_virtualenv(tmp_path_factory): builder.create(virtualenv_at) venv_for_virtualenv = CURRENT.find_exe_based_of(inside_folder=virtualenv_at) cmd = venv_for_virtualenv, "-m", "pip", "install", "virtualenv==16.6.1" - subprocess.check_call(cmd) + process = Popen(cmd) + _, __ = process.communicate() + assert not process.returncode virtualenv_python = tmp_path_factory.mktemp("virtualenv") cmd = venv_for_virtualenv, "-m", "virtualenv", virtualenv_python - subprocess.check_call(cmd) + process = Popen(cmd) + _, __ = process.communicate() + assert not process.returncode return CURRENT.find_exe_based_of(inside_folder=virtualenv_python) diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 374d5774c..60b6b6977 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -57,32 +57,28 @@ def test_destination_not_write_able(tmp_path, capsys): target.chmod(prev_mod) -SYSTEM = get_env_debug_info(CURRENT.system_executable, DEBUG_SCRIPT) +SYSTEM = get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT) -def cleanup_sys_path(path): +def cleanup_sys_path(paths): from virtualenv.interpreters.create.creator import HERE - path = [Path(i).absolute() for i in path] + paths = [Path(i).absolute() for i in paths] to_remove = [Path(HERE)] if str("PYCHARM_HELPERS_DIR") in os.environ: - to_remove.append(Path(os.environ[str("PYCHARM_HELPERS_DIR")]).parent / "pydev") - for elem in to_remove: - try: - index = path.index(elem) - del path[index] - except ValueError: - pass - return path + to_remove.append(Path(os.environ[str("PYCHARM_HELPERS_DIR")]).parent) + to_remove.append(Path(os.path.expanduser("~")) / ".PyCharm") + result = [i for i in paths if not any(str(i).startswith(str(t)) for t in to_remove)] + return result @pytest.mark.parametrize("global_access", [False, True], ids=["no_global", "ok_global"]) @pytest.mark.parametrize( "use_venv", [False, True] if six.PY3 else [False], ids=["no_venv", "venv"] if six.PY3 else ["no_venv"] ) -def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, special_char_name): - dest = tmp_path / special_char_name - cmd = ["-v", "-v", "-p", str(python), str(dest), "--without-pip", "--activators", ""] +def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, special_name_dir): + dest = special_name_dir + cmd = ["-v", "-v", "-p", six.ensure_text(python), six.ensure_text(str(dest)), "--without-pip", "--activators", ""] if global_access: cmd.append("--system-site-packages") if use_venv: @@ -92,17 +88,20 @@ def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, for site_package in result.creator.site_packages: content = list(site_package.iterdir()) assert not content, "\n".join(str(i) for i in content) - assert result.creator.env_name == special_char_name - sys_path = cleanup_sys_path(result.creator.debug["sys"]["path"]) + assert result.creator.env_name == dest.name + debug = result.creator.debug + sys_path = cleanup_sys_path(debug["sys"]["path"]) system_sys_path = cleanup_sys_path(SYSTEM["sys"]["path"]) our_paths = set(sys_path) - set(system_sys_path) - our_paths_repr = "\n".join(repr(i) for i in our_paths) + our_paths_repr = "\n".join(six.ensure_text(repr(i)) for i in our_paths) # ensure we have at least one extra path added assert len(our_paths) >= 1, our_paths_repr # ensure all additional paths are related to the virtual environment for path in our_paths: - assert str(path).startswith(str(dest)), path + assert str(path).startswith(str(dest)), "{} does not start with {}".format( + six.ensure_text(str(path)), six.ensure_text(str(dest)) + ) # ensure there's at least a site-packages folder as part of the virtual environment added assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr diff --git a/tox.ini b/tox.ini index 0731e1038..16a869906 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ force_grid_wrap = 0 line_length = 120 known_standard_library = ConfigParser known_first_party = virtualenv -known_third_party = appdirs,coverage,entrypoints,git,packaging,pathlib2,pytest,setuptools,six +known_third_party = _subprocess,appdirs,coverage,entrypoints,git,packaging,pathlib2,pytest,setuptools,six [flake8] max-complexity = 22 From 0fd241d6ed13063df83d5a64f7842ad8eac9fa35 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 2 Jan 2020 14:36:34 +0000 Subject: [PATCH 16/18] do not single out mbcs, but the file system encoder --- .../interpreters/create/cpython/cpython2.py | 15 --------------- src/virtualenv/interpreters/create/creator.py | 15 ++++++++++++++- tests/conftest.py | 6 ++++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/virtualenv/interpreters/create/cpython/cpython2.py b/src/virtualenv/interpreters/create/cpython/cpython2.py index bacefeeb9..7c7915882 100644 --- a/src/virtualenv/interpreters/create/cpython/cpython2.py +++ b/src/virtualenv/interpreters/create/cpython/cpython2.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, unicode_literals import abc -from argparse import ArgumentTypeError import six @@ -68,17 +67,3 @@ def add_folder(self, folder): class CPython2Windows(CPython2, CPythonWindows): """CPython 2 on Windows""" - - @classmethod - def validate_dest_dir(cls, raw_value): - # the python prefix discovery mechanism on Windows python 2 uses mbcs - so anything that's not encode-able - # needs to be refused - path_converted = raw_value.encode("mbcs", errors="ignore").decode("mbcs") - if path_converted != raw_value: - refused = set(raw_value) - { - c for c, i in ((char, char.encode("mbcs")) for char in raw_value) if c == "?" or i != "?" - } - raise ArgumentTypeError( - "mbcs (path encoder for CPython2.7 on Windows) does not support characters %r".format(refused) - ) - return super(CPython2Windows, cls).validate_dest_dir(raw_value) diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index 69aec645a..325fb2e3d 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -54,13 +54,26 @@ def add_parser_arguments(cls, parser, interpreter): @classmethod def validate_dest_dir(cls, raw_value): + """No path separator in the path, valid chars and must be write-able""" + def non_write_able(dest, value): common = Path(*os.path.commonprefix([value.parts, dest.parts])) raise ArgumentTypeError( "the destination {} is not write-able at {}".format(dest.relative_to(common), common) ) - """No path separator in the path and must be write-able""" + # the file system must be able to encode + encoding = sys.getfilesystemencoding() + path_converted = raw_value.encode(encoding, errors="ignore").decode(encoding) + if path_converted != raw_value: + refused = set(raw_value) - { + c + for c, i in ((char, char.encode(encoding)) for char in raw_value) + if c == "?" or i != six.ensure_str("?") + } + raise ArgumentTypeError( + "the file system codec ({}) does not support characters {!r}".format(encoding, refused) + ) if os.pathsep in raw_value: raise ArgumentTypeError( "destination {!r} must not contain the path separator ({}) as this would break " diff --git a/tests/conftest.py b/tests/conftest.py index 0ff2c1f58..1cb76f849 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -227,14 +227,16 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): base = "$ Γ¨Ρ€Ρ‚πŸš’β™žδΈ­η‰‡" + encoding = sys.getfilesystemencoding() if sys.platform == "win32" and six.PY2: - # PY2 windows uses mbcs as path encoder, so don't try to use what's not encode-able by that + # let's not include characters that the file system cannot encode) result = "" for char in base: - encoded = char.encode("mbcs", errors="ignore") + encoded = char.encode(encoding, errors="ignore") if char == "?" or encoded != six.ensure_str("?"): result += char base = result + assert base return base From f89d2bf8daf851efc043d8c98c37c2901f33bd3c Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 2 Jan 2020 14:37:46 +0000 Subject: [PATCH 17/18] do not install pathlib python 2 windows Signed-off-by: Bernat Gabor --- setup.cfg | 2 +- src/virtualenv/interpreters/create/creator.py | 1 + tests/unit/interpreters/create/test_creator.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 45694e863..bd39b6704 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = six >= 1.12.0, < 2 appdirs >= 1.4.3 entrypoints >= 0.3, <1 - pathlib2 >= 2.3.3, < 3; python_version < '3.4' + pathlib2 >= 2.3.3, < 3; python_version < '3.4' and sys.platform != 'win32' distlib >= 0.3.0, <1; sys.platform == 'win32' [options.packages.find] where = src diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py index 325fb2e3d..b917a3709 100644 --- a/src/virtualenv/interpreters/create/creator.py +++ b/src/virtualenv/interpreters/create/creator.py @@ -63,6 +63,7 @@ def non_write_able(dest, value): ) # the file system must be able to encode + # note in newer CPython this is always utf-8 https://www.python.org/dev/peps/pep-0529/ encoding = sys.getfilesystemencoding() path_converted = raw_value.encode(encoding, errors="ignore").decode(encoding) if path_converted != raw_value: diff --git a/tests/unit/interpreters/create/test_creator.py b/tests/unit/interpreters/create/test_creator.py index 60b6b6977..4db7a0bfb 100644 --- a/tests/unit/interpreters/create/test_creator.py +++ b/tests/unit/interpreters/create/test_creator.py @@ -88,7 +88,7 @@ def test_create_no_seed(python, use_venv, global_access, tmp_path, coverage_env, for site_package in result.creator.site_packages: content = list(site_package.iterdir()) assert not content, "\n".join(str(i) for i in content) - assert result.creator.env_name == dest.name + assert result.creator.env_name == six.ensure_text(dest.name) debug = result.creator.debug sys_path = cleanup_sys_path(debug["sys"]["path"]) system_sys_path = cleanup_sys_path(SYSTEM["sys"]["path"]) From 93fdbb72f9683a6c5dcf9b9fab57b99e5e98f197 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 2 Jan 2020 15:18:57 +0000 Subject: [PATCH 18/18] fix encoding on py35 Signed-off-by: Bernat Gabor --- tests/conftest.py | 19 ++++++++++--------- .../interpreters/discovery/test_discovery.py | 10 ++++++---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1cb76f849..c451c2bf8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -228,16 +228,17 @@ def is_inside_ci(): def special_char_name(): base = "$ Γ¨Ρ€Ρ‚πŸš’β™žδΈ­η‰‡" encoding = sys.getfilesystemencoding() - if sys.platform == "win32" and six.PY2: - # let's not include characters that the file system cannot encode) - result = "" - for char in base: - encoded = char.encode(encoding, errors="ignore") - if char == "?" or encoded != six.ensure_str("?"): + # let's not include characters that the file system cannot encode) + result = "" + for char in base: + try: + encoded = char.encode(encoding, errors="strict") + if char == "?" or encoded != b"?": # mbcs notably on Python 2 uses replace even for strict result += char - base = result - assert base - return base + except ValueError: + continue + assert result + return result @pytest.fixture() diff --git a/tests/unit/interpreters/discovery/test_discovery.py b/tests/unit/interpreters/discovery/test_discovery.py index 4863252ef..f66a5b099 100644 --- a/tests/unit/interpreters/discovery/test_discovery.py +++ b/tests/unit/interpreters/discovery/test_discovery.py @@ -5,6 +5,7 @@ from uuid import uuid4 import pytest +import six from virtualenv.interpreters.discovery.builtin import get_interpreter from virtualenv.interpreters.discovery.py_info import CURRENT @@ -12,7 +13,7 @@ @pytest.mark.skipif(sys.platform == "win32", reason="symlink is not guaranteed to work on windows") @pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) -def test_discovery_via_path(tmp_path, monkeypatch, case): +def test_discovery_via_path(monkeypatch, case, special_name_dir): core = "somethingVeryCryptic{}".format(".".join(str(i) for i in CURRENT.version_info[0:3])) name = "somethingVeryCryptic" if case == "lower": @@ -20,9 +21,10 @@ def test_discovery_via_path(tmp_path, monkeypatch, case): elif case == "upper": name = name.upper() exe_name = "{}{}{}".format(name, CURRENT.version_info.major, ".exe" if sys.platform == "win32" else "") - executable = tmp_path / exe_name - os.symlink(sys.executable, str(executable)) - new_path = os.pathsep.join([str(tmp_path)] + os.environ.get(str("PATH"), str("")).split(os.pathsep)) + special_name_dir.mkdir() + executable = special_name_dir / exe_name + os.symlink(sys.executable, six.ensure_text(str(executable))) + new_path = os.pathsep.join([str(special_name_dir)] + os.environ.get(str("PATH"), str("")).split(os.pathsep)) monkeypatch.setenv(str("PATH"), new_path) interpreter = get_interpreter(core)