From 9f59811704015f52f932a23b85769a96318d921d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 20:26:59 -0400 Subject: [PATCH 01/10] Improve import times and runtimes - Lazily import requirementslib Signed-off-by: Dan Ryan --- news/2485.feature | 1 + pipenv/utils.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 news/2485.feature diff --git a/news/2485.feature b/news/2485.feature new file mode 100644 index 0000000000..f52a6fe4b8 --- /dev/null +++ b/news/2485.feature @@ -0,0 +1 @@ +Improved import times and CLI runtimes with minor tweaks. diff --git a/pipenv/utils.py b/pipenv/utils.py index d468afcb03..3b906d02ca 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -49,7 +49,6 @@ def detach(self): from collections.abc import Mapping except ImportError: from collections import Mapping -from .vendor.requirementslib import Requirement if six.PY2: @@ -230,6 +229,7 @@ def actually_resolve_deps( from pipenv.patched.piptools.scripts.compile import get_pip_command from pipenv.patched.piptools import logging as piptools_logging from pipenv.patched.piptools.exceptions import NoCandidateFound + from .vendor.requirementslib import Requirement from ._compat import TemporaryDirectory, NamedTemporaryFile class PipCommand(basecommand.Command): @@ -1162,6 +1162,7 @@ def get_vcs_deps( ): from .patched.notpip._internal.vcs import VcsSupport from ._compat import TemporaryDirectory, Path + from .vendor.requirementslib import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" reqs = [] From 5746e9af8530ed2429a3934dbe67961ff3bc8fc5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 21:02:26 -0400 Subject: [PATCH 02/10] More improvements Signed-off-by: Dan Ryan --- pipenv/core.py | 13 +++++++------ pipenv/project.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index ea8a0698c3..dec98d3667 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -4,18 +4,15 @@ import os import sys import shutil -import signal import time import tempfile from glob import glob import json as simplejson - import click import click_completion import crayons import dotenv import delegator -from .vendor import pexpect from first import first import pipfile from blindspin import spinner @@ -25,7 +22,6 @@ from .cmdparse import ScriptEmptyError from .project import Project, SourceNotFound -from .vendor.requirementslib import Requirement from .utils import ( convert_deps_to_pip, is_required_version, @@ -973,6 +969,7 @@ def parse_download_fname(fname, name): def get_downloads_info(names_map, section): + from .vendor.requirementslib import Requirement info = [] p = project.parsed_pipfile for fname in os.listdir(project.download_location): @@ -1005,6 +1002,7 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" + from .vendor.requirementslib import Requirement from .utils import get_vcs_deps cached_lockfile = {} if not pre: @@ -1364,7 +1362,7 @@ def pip_install( ): from notpip._internal import logger as piplogger from notpip._vendor.pyparsing import ParseException - + from .vendor.requirementslib import Requirement if verbose: click.echo( crayons.normal('Installing {0!r}'.format(package_name), bold=True), @@ -1686,6 +1684,7 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): + from .vendor.requirementslib import Requirement packages = {} results = delegator.run('{0} freeze'.format(which('pip'))).out.strip( ).split( @@ -1912,6 +1911,7 @@ def do_install( # We should do this part first to make sure that we actually do selectively upgrade # the items specified if selective_upgrade: + from .vendor.requirementslib import Requirement for i, package_name in enumerate(package_names[:]): section = project.packages if not dev else project.dev_packages package = Requirement.from_line(package_name) @@ -1953,6 +1953,7 @@ def do_install( # This is for if the user passed in dependencies, then we want to maek sure we else: + from .vendor.requirementslib import Requirement for package_name in package_names: click.echo( crayons.normal( @@ -2531,9 +2532,9 @@ def do_clean( ctx, three=None, python=None, dry_run=False, bare=False, verbose=False, pypi_mirror=None ): # Ensure that virtualenv is available. + from .vendor.requirementslib import Requirement ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) - installed_package_names = [] pip_freeze_command = delegator.run('{0} freeze'.format(which_pip())) for line in pip_freeze_command.out.split('\n'): diff --git a/pipenv/project.py b/pipenv/project.py index 9c8901e8c0..03d160fdb8 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -17,7 +17,6 @@ from ._compat import Path from .cmdparse import Script -from .vendor.requirementslib import Requirement from .utils import ( atomic_open_for_write, mkdir_p, @@ -728,6 +727,7 @@ def remove_package_from_pipfile(self, package_name, dev=False): self.write_toml(p) def add_package_to_pipfile(self, package_name, dev=False): + from .vendor.requirementslib import Requirement # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. From 9f71a1bfbe7a3c21588fb58b67509fa1e1991e58 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 21:39:46 -0400 Subject: [PATCH 03/10] Reimplement `get_workon_home` to save import - Pew import is super expensive - Fixes #2843 Signed-off-by: Dan Ryan --- pipenv/_compat.py | 3 --- pipenv/core.py | 6 +++--- pipenv/project.py | 2 +- pipenv/utils.py | 11 +++++++++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pipenv/_compat.py b/pipenv/_compat.py index fa9e0a1d9e..96026089d5 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -78,9 +78,6 @@ def pip_import(module_path, subimport=None, old_path=None): return _tmp -vcs = pip_import('vcs', 'VcsSupport') - - class TemporaryDirectory(object): """Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For diff --git a/pipenv/core.py b/pipenv/core.py index dec98d3667..fbc2ae54eb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -16,11 +16,8 @@ from first import first import pipfile from blindspin import spinner -from requests.packages import urllib3 -from requests.packages.urllib3.exceptions import InsecureRequestWarning import six -from .cmdparse import ScriptEmptyError from .project import Project, SourceNotFound from .utils import ( convert_deps_to_pip, @@ -135,6 +132,8 @@ def which(command, location=None, allow_global=False): # Disable warnings for Python 2.6. if 'urllib3' in globals(): + import urllib3 + from urllib3.exceptions import InsecureRequestWarning urllib3.disable_warnings(InsecureRequestWarning) project = Project(which=which) @@ -2247,6 +2246,7 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): Args are appended to the command in [scripts] section of project if found. """ + from .cmdparse import ScriptEmptyError # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) load_dot_env() diff --git a/pipenv/project.py b/pipenv/project.py index 03d160fdb8..a0841ce4c7 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -32,6 +32,7 @@ python_version, safe_expandvars, is_star, + get_workon_home, ) from .environments import ( PIPENV_MAX_DEPTH, @@ -240,7 +241,6 @@ def virtualenv_exists(self): @classmethod def _get_virtualenv_location(cls, name): - from .patched.pew.pew import get_workon_home venv = get_workon_home() / name if not venv.exists(): return '' diff --git a/pipenv/utils.py b/pipenv/utils.py index 3b906d02ca..ca5f3d5b36 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1279,3 +1279,14 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def get_workon_home(): + from ._compat import Path + workon_home = os.environ.get('WORKON_HOME') + if not workon_home: + if os.name == 'nt': + workon_home = '~/.virtualenvs' + else: + workon_home = os.sep.join([os.environ.get('XDG_DATA_HOME', '~/.local/share'), 'virtualenvs']) + return Path(os.path.expandvars(workon_home)).expanduser() From 99b0a75b5907f2a48d7ca7ec7fe505f184f30264 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 21:53:08 -0400 Subject: [PATCH 04/10] Add missed import Signed-off-by: Dan Ryan --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index fbc2ae54eb..a7cc851df6 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1001,7 +1001,6 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - from .vendor.requirementslib import Requirement from .utils import get_vcs_deps cached_lockfile = {} if not pre: @@ -1164,6 +1163,7 @@ def do_lock( def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): """Executes the purge functionality.""" + from .vendor.requirementslib.models.requirements import Requirement if downloads: if not bare: click.echo( From 547110b7b4268126e22307d02beb79ef4e8d21b4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 30 Jun 2018 22:32:59 -0400 Subject: [PATCH 05/10] Add additional comparison for markers Signed-off-by: Dan Ryan --- tests/integration/test_install_markers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 42401c159b..ee83179b45 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -149,7 +149,8 @@ def test_resolver_unique_markers(PipenvInstance, pypi): assert 'yarl' in p.lockfile['default'] yarl = p.lockfile['default']['yarl'] assert 'markers' in yarl - assert yarl['markers'] == "python_version in '3.4, 3.5, 3.6'" + # Two possible marker sets are ok here + assert yarl['markers'] in ["python_version in '3.4, 3.5, 3.6'", "python_version >= '3.4.1'"] @pytest.mark.project From 55f02af49bfd3a2d1c3e8a9d3539f54deb3acf70 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 1 Jul 2018 19:12:15 +0800 Subject: [PATCH 06/10] Format --- pipenv/core.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index a7cc851df6..22c386069e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -13,7 +13,6 @@ import crayons import dotenv import delegator -from first import first import pipfile from blindspin import spinner import six @@ -31,7 +30,6 @@ python_version, find_windows_executable, prepare_pip_source_args, - temp_environ, is_valid_url, is_pypi_url, create_mirror_source, @@ -47,7 +45,7 @@ TemporaryDirectory, Path ) -from .import pep508checker, progress +from . import pep508checker, progress from .environments import ( PIPENV_COLORBLIND, PIPENV_NOSPIN, @@ -66,9 +64,7 @@ SESSION_IS_INTERACTIVE, PIPENV_USE_SYSTEM, PIPENV_DOTENV_LOCATION, - PIPENV_SHELL, PIPENV_PYTHON, - PIPENV_VIRTUALENV, PIPENV_CACHE_DIR, ) @@ -2142,7 +2138,6 @@ def do_uninstall( do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) - def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) From 9930d2edbbd14ad9019e43745fdd84764c04f2c9 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 1 Jul 2018 19:13:21 +0800 Subject: [PATCH 07/10] Remove Python 2.6 compat --- pipenv/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 22c386069e..d6db68b3ea 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -126,11 +126,6 @@ def which(command, location=None, allow_global=False): return p -# Disable warnings for Python 2.6. -if 'urllib3' in globals(): - import urllib3 - from urllib3.exceptions import InsecureRequestWarning - urllib3.disable_warnings(InsecureRequestWarning) project = Project(which=which) From f998d017e6cab96255425d33bb1d359b1df37b58 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 1 Jul 2018 19:20:36 +0800 Subject: [PATCH 08/10] Comment and format --- pipenv/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index ca5f3d5b36..60c12cbd4a 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1281,6 +1281,7 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +# Duplicated from Pew to avoid importing it (performance considerations). def get_workon_home(): from ._compat import Path workon_home = os.environ.get('WORKON_HOME') @@ -1288,5 +1289,8 @@ def get_workon_home(): if os.name == 'nt': workon_home = '~/.virtualenvs' else: - workon_home = os.sep.join([os.environ.get('XDG_DATA_HOME', '~/.local/share'), 'virtualenvs']) + workon_home = os.path.join( + os.environ.get('XDG_DATA_HOME', '~/.local/share'), + 'virtualenvs', + ) return Path(os.path.expandvars(workon_home)).expanduser() From 7050b4c2c6dca82186a380261606947712ce57b8 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sun, 1 Jul 2018 20:03:00 +0800 Subject: [PATCH 09/10] Global shouldn't be needed --- pipenv/core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d6db68b3ea..01586f8c78 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -66,6 +66,7 @@ PIPENV_DOTENV_LOCATION, PIPENV_PYTHON, PIPENV_CACHE_DIR, + PIPENV_VIRTUALENV, ) # Packages that should be ignored later. @@ -241,7 +242,6 @@ def import_from_code(path='.'): def ensure_pipfile(validate=True, skip_requirements=False, system=False): """Creates a Pipfile for the project, if it doesn't exist.""" - global USING_DEFAULT_PYTHON, PIPENV_VIRTUALENV # Assert Pipfile exists. python = which('python') if not (USING_DEFAULT_PYTHON or system) else None if project.pipfile_is_empty: @@ -1223,7 +1223,6 @@ def do_init( ): """Executes the init functionality.""" cleanup_reqdir = False - global PIPENV_VIRTUALENV if not system: if not project.virtualenv_exists: try: @@ -1762,8 +1761,7 @@ def do_install( keep_outdated = project.settings.get('keep_outdated') remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. - global PIPENV_VIRTUALENV - if system and package_name and not PIPENV_VIRTUALENV: + if (system and package_name) and not (PIPENV_VIRTUALENV): click.echo( '{0}: --system is intended to be used for Pipfile installation, ' 'not installation of specific packages. Aborting.'.format( From a245be18196fb4152cae252c276bfd309c47ee38 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 1 Jul 2018 14:16:48 -0400 Subject: [PATCH 10/10] Use more local tests Signed-off-by: Dan Ryan --- MANIFEST.in | 2 +- pipenv/cli.py | 2 +- pipenv/core.py | 39 ++++++++++++++++++++++----------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 61327a1b8b..3d3588d2b4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include README.rst LICENSE NOTICES HISTORY.txt pipenv/patched/safety.zip +include README.rst CHANGELOG.rst LICENSE NOTICES HISTORY.txt pipenv/patched/safety.zip include pipenv/vendor/pipreqs/stdlib include pipenv/vendor/pipreqs/mapping include pipenv/vendor/click_completion/*.j2 diff --git a/pipenv/cli.py b/pipenv/cli.py index 646cd0ac8c..86a28ccf9c 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -269,7 +269,7 @@ def cli( loc = project.virtualenv_location echo( crayons.normal( - u'{0} ({1})…'.format( + u'{0} ({1})...'.format( crayons.normal('Removing virtualenv', bold=True), crayons.green(loc), ) diff --git a/pipenv/core.py b/pipenv/core.py index 01586f8c78..ec585dfecf 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -62,11 +62,8 @@ PIPENV_MAX_SUBPROCESS, PIPENV_DONT_USE_PYENV, SESSION_IS_INTERACTIVE, - PIPENV_USE_SYSTEM, PIPENV_DOTENV_LOCATION, - PIPENV_PYTHON, PIPENV_CACHE_DIR, - PIPENV_VIRTUALENV, ) # Packages that should be ignored later. @@ -242,6 +239,7 @@ def import_from_code(path='.'): def ensure_pipfile(validate=True, skip_requirements=False, system=False): """Creates a Pipfile for the project, if it doesn't exist.""" + from .environments import PIPENV_VIRTUALENV # Assert Pipfile exists. python = which('python') if not (USING_DEFAULT_PYTHON or system) else None if project.pipfile_is_empty: @@ -376,6 +374,7 @@ def find_a_system_python(python): def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. + from .environments import PIPENV_PYTHON if PIPENV_PYTHON and python is False and three is None: python = PIPENV_PYTHON @@ -532,6 +531,7 @@ def activate_pyenv(): def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv, if one doesn't exist.""" + from .environments import PIPENV_USE_SYSTEM def abort(): sys.exit(1) @@ -597,6 +597,7 @@ def ensure_project( pypi_mirror=None, ): """Ensures both Pipfile and virtualenv exist for the project.""" + from .environments import PIPENV_USE_SYSTEM # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True @@ -1222,6 +1223,7 @@ def do_init( pypi_mirror=None, ): """Executes the init functionality.""" + from .environments import PIPENV_VIRTUALENV cleanup_reqdir = False if not system: if not project.virtualenv_exists: @@ -1626,20 +1628,21 @@ def gen(out): def warn_in_virtualenv(): - if PIPENV_USE_SYSTEM: - # Only warn if pipenv isn't already active. - if 'PIPENV_ACTIVE' not in os.environ: - click.echo( - '{0}: Pipenv found itself running within a virtual environment, ' - 'so it will automatically use that environment, instead of ' - 'creating its own for any project. You can set ' - '{1} to force pipenv to ignore that environment and create ' - 'its own instead.'.format( - crayons.green('Courtesy Notice'), - crayons.normal('PIPENV_IGNORE_VIRTUALENVS=1', bold=True), - ), - err=True, - ) + from .environments import PIPENV_USE_SYSTEM, PIPENV_VIRTUALENV + # Only warn if pipenv isn't already active. + pipenv_active = os.environ.get('PIPENV_ACTIVE') + if (PIPENV_USE_SYSTEM or PIPENV_VIRTUALENV) and not pipenv_active: + click.echo( + '{0}: Pipenv found itself running within a virtual environment, ' + 'so it will automatically use that environment, instead of ' + 'creating its own for any project. You can set ' + '{1} to force pipenv to ignore that environment and create ' + 'its own instead.'.format( + crayons.green('Courtesy Notice'), + crayons.normal('PIPENV_IGNORE_VIRTUALENVS=1', bold=True), + ), + err=True, + ) def ensure_lockfile(keep_outdated=False, pypi_mirror=None): @@ -1730,6 +1733,7 @@ def do_install( keep_outdated=False, selective_upgrade=False, ): + from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from notpip._internal.exceptions import PipError requirements_directory = TemporaryDirectory( @@ -2058,6 +2062,7 @@ def do_uninstall( keep_outdated=False, pypi_mirror=None, ): + from .environments import PIPENV_USE_SYSTEM # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True