From 1103043bc31cbcefe83fab5fcce3606df8569260 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 14 Mar 2018 17:22:40 -0400 Subject: [PATCH] black Signed-off-by: Kenneth Reitz --- Pipfile | 4 +- Pipfile.lock | 6 +- pipenv/core.py | 1210 +++++++++++++++++++++----------------------- pipenv/progress.py | 397 ++++++++------- pipenv/project.py | 156 +++--- pipenv/utils.py | 356 +++++++------ 6 files changed, 1053 insertions(+), 1076 deletions(-) diff --git a/Pipfile b/Pipfile index 75f94a51be..61cf8d6a44 100644 --- a/Pipfile +++ b/Pipfile @@ -12,11 +12,13 @@ click = "*" pytest-pypy = {path = "./tests/pytest-pypi", editable = true} pytest-tap = "*" stdeb = {version="*", sys_platform="== 'linux'"} -black = {git = "https://github.com/ambv/black.git", python_version = "=='3.6'"} +black = {git = "https://github.com/ambv/black.git", editable = true} + [packages] + [scripts] tests = "bash ./run-tests.sh" diff --git a/Pipfile.lock b/Pipfile.lock index a538c3d7f2..ee6dade3fd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6b77ca7e54fbb6febca620c0fb96c0b6ba17aed618c1eaa7e7cba8604a1b3f84" + "sha256": "014daa4b632edc332ac864fe8220872044c779155e75f3c0ef7e3a832b49001f" }, "pipfile-spec": 6, "requires": {}, @@ -44,8 +44,8 @@ "version": "==2.5.3" }, "black": { - "git": "https://github.com/ambv/black.git", - "python_version": "=='3.6'" + "editable": true, + "git": "https://github.com/ambv/black.git" }, "certifi": { "hashes": [ diff --git a/pipenv/core.py b/pipenv/core.py index 82d30ae24f..76eab37b41 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import contextlib import codecs import logging @@ -31,22 +30,53 @@ from .project import Project from .utils import ( - convert_deps_from_pip, convert_deps_to_pip, is_required_version, - proper_case, pep423_name, split_file, merge_deps, venv_resolve_deps, escape_grouped_arguments, is_vcs, - python_version, find_windows_executable, is_file, prepare_pip_source_args, - temp_environ, is_valid_url, download_file, get_requirement, need_update_check, - touch_update_stamp, is_pinned, is_star, TemporaryDirectory + convert_deps_from_pip, + convert_deps_to_pip, + is_required_version, + proper_case, + pep423_name, + split_file, + merge_deps, + venv_resolve_deps, + escape_grouped_arguments, + is_vcs, + python_version, + find_windows_executable, + is_file, + prepare_pip_source_args, + temp_environ, + is_valid_url, + download_file, + get_requirement, + need_update_check, + touch_update_stamp, + is_pinned, + is_star, + TemporaryDirectory, ) from .__version__ import __version__ -from . import pep508checker, progress +from .import pep508checker, progress from .environments import ( - PIPENV_COLORBLIND, PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_VENV_IN_PROJECT, PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, - PIPENV_HIDE_EMOJIS, PIPENV_INSTALL_TIMEOUT, PYENV_ROOT, - PYENV_INSTALLED, PIPENV_YES, PIPENV_DONT_LOAD_ENV, - PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_MAX_SUBPROCESS, - PIPENV_DONT_USE_PYENV, SESSION_IS_INTERACTIVE, PIPENV_USE_SYSTEM, - PIPENV_DOTENV_LOCATION, PIPENV_SHELL, PIPENV_PYTHON + PIPENV_COLORBLIND, + PIPENV_NOSPIN, + PIPENV_SHELL_FANCY, + PIPENV_VENV_IN_PROJECT, + PIPENV_TIMEOUT, + PIPENV_SKIP_VALIDATION, + PIPENV_HIDE_EMOJIS, + PIPENV_INSTALL_TIMEOUT, + PYENV_ROOT, + PYENV_INSTALLED, + PIPENV_YES, + PIPENV_DONT_LOAD_ENV, + PIPENV_DEFAULT_PYTHON_VERSION, + PIPENV_MAX_SUBPROCESS, + PIPENV_DONT_USE_PYENV, + SESSION_IS_INTERACTIVE, + PIPENV_USE_SYSTEM, + PIPENV_DOTENV_LOCATION, + PIPENV_SHELL, + PIPENV_PYTHON, ) # Backport required for earlier versions of Python. @@ -54,45 +84,38 @@ from backports.shutil_get_terminal_size import get_terminal_size else: from shutil import get_terminal_size - # Packages that should be ignored later. -BAD_PACKAGES = ( - 'setuptools', 'pip', 'wheel', 'packaging', 'distribute', -) - +BAD_PACKAGES = ('setuptools', 'pip', 'wheel', 'packaging', 'distribute') # Are we using the default Python? USING_DEFAULT_PYTHON = True - if not PIPENV_HIDE_EMOJIS: now = time.localtime() - # Halloween easter-egg. - if ((now.tm_mon == 10) and (now.tm_mday == 30)) or ((now.tm_mon == 10) and (now.tm_mday == 31)): + if ((now.tm_mon == 10) and (now.tm_mday == 30)) or ( + (now.tm_mon == 10) and (now.tm_mday == 31) + ): INSTALL_LABEL = 'šŸŽƒ ' - # Christmas easter-egg. - elif ((now.tm_mon == 12) and (now.tm_mday == 24)) or ((now.tm_mon == 12) and (now.tm_mday == 25)): + elif ((now.tm_mon == 12) and (now.tm_mday == 24)) or ( + (now.tm_mon == 12) and (now.tm_mday == 25) + ): INSTALL_LABEL = 'šŸŽ… ' - else: INSTALL_LABEL = 'šŸ ' - INSTALL_LABEL2 = crayons.normal('ā˜¤ ', bold=True) STARTING_LABEL = ' ' else: INSTALL_LABEL = ' ' INSTALL_LABEL2 = ' ' STARTING_LABEL = ' ' - # Enable shell completion. click_completion.init() - # Disable colors, for the soulless. if PIPENV_COLORBLIND: crayons.disable() - # Disable spinner, for cleaner build logs (the unworthy). if PIPENV_NOSPIN: + @contextlib.contextmanager # noqa: F811 def spinner(): yield @@ -101,7 +124,6 @@ def spinner(): def which(command, location=None, allow_global=False): if location is None: location = project.virtualenv_location - if not allow_global: if os.name == 'nt': p = find_windows_executable(os.path.join(location, 'Scripts'), command) @@ -110,14 +132,12 @@ def which(command, location=None, allow_global=False): else: if command == 'python': p = sys.executable - return p # Disable warnings for Python 2.6. if 'urllib3' in globals(): urllib3.disable_warnings(InsecureRequestWarning) - project = Project(which=which) @@ -126,10 +146,14 @@ def load_dot_env(): if not PIPENV_DONT_LOAD_ENV: # If the project doesn't exist yet, check current directory for a .env file project_directory = project.project_directory or '.' - - denv = dotenv.find_dotenv(PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, '.env'])) + denv = dotenv.find_dotenv( + PIPENV_DOTENV_LOCATION or os.sep.join([project_directory, '.env']) + ) if os.path.isfile(denv): - click.echo(crayons.normal('Loading .env environment variablesā€¦', bold=True), err=True) + click.echo( + crayons.normal('Loading .env environment variablesā€¦', bold=True), + err=True, + ) dotenv.load_dotenv(denv, override=True) @@ -149,41 +173,38 @@ def ensure_latest_self(user=False): sys.exit(1) latest = max(map(semver.parse_version_info, r.json()['releases'].keys())) current = semver.parse_version_info(__version__) - if current < latest: - import site - click.echo('{0}: {1} is now available. Automatically upgrading!'.format( - crayons.green('Courtesy Notice'), - crayons.yellow('Pipenv {v.major}.{v.minor}.{v.patch}'.format(v=latest)), - ), err=True) - + click.echo( + '{0}: {1} is now available. Automatically upgrading!'.format( + crayons.green('Courtesy Notice'), + crayons.yellow('Pipenv {v.major}.{v.minor}.{v.patch}'.format(v=latest)), + ), + err=True, + ) # Resolve user site, enable user mode automatically. if site.ENABLE_USER_SITE and site.USER_SITE in sys.modules['pipenv'].__file__: args = ['install', '--user', '--upgrade', 'pipenv', '--no-cache'] else: args = ['install', '--upgrade', 'pipenv', '--no-cache'] - os.environ['PIP_PYTHON_VERSION'] = str('.'.join(map(str, sys.version_info[:3]))) os.environ['PIP_PYTHON_PATH'] = str(sys.executable) - sys.modules['pip'].main(args) - - click.echo('{0} to {1}!'.format( - crayons.green('Pipenv updated'), - crayons.yellow('{v.major}.{v.minor}.{v.patch}'.format(v=latest)) - )) + click.echo( + '{0} to {1}!'.format( + crayons.green('Pipenv updated'), + crayons.yellow('{v.major}.{v.minor}.{v.patch}'.format(v=latest)), + ) + ) else: click.echo(crayons.green('All good!')) def cleanup_virtualenv(bare=True): """Removes the virtualenv directory from the system.""" - if not bare: click.echo(crayons.red('Environment creation aborted.')) - try: # Delete the virtualenv. shutil.rmtree(project.virtualenv_location, ignore_errors=True) @@ -193,19 +214,24 @@ def cleanup_virtualenv(bare=True): def ensure_latest_pip(): """Updates pip to the latest version.""" - # Ensure that pip is installed. try: - c = delegator.run('{0} install pip'.format(escape_grouped_arguments(which_pip()))) - + c = delegator.run( + '{0} install pip'.format(escape_grouped_arguments(which_pip())) + ) # Check if version is out of date. if 'however' in c.err: # If version is out of date, update. - click.echo(crayons.normal(u'Pip is out of dateā€¦ updating to latest.', bold=True)) - + click.echo( + crayons.normal(u'Pip is out of dateā€¦ updating to latest.', bold=True) + ) windows = '-m' if os.name == 'nt' else '' - - c = delegator.run('{0} install {1} pip --upgrade'.format(escape_grouped_arguments(which_pip()), windows), block=False) + c = delegator.run( + '{0} install {1} pip --upgrade'.format( + escape_grouped_arguments(which_pip()), windows + ), + block=False, + ) click.echo(crayons.blue(c.out)) except AttributeError: pass @@ -214,43 +240,36 @@ def ensure_latest_pip(): def import_requirements(r=None, dev=False): import pip9 from pip9.req.req_file import parse_requirements + # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. # Since we're not making any network calls, it's initialized to nothing. - if r: assert os.path.isfile(r) - # Default path, if none is provided. if r is None: r = project.requirements_location - with open(r, 'r') as f: contents = f.read() - indexes = [] # Find and add extra indexes. for line in contents.split('\n'): if line.startswith(('-i ', '--index ', '--index-url ')): indexes.append(line.split()[1]) - reqs = [f for f in parse_requirements(r, session=pip9._vendor.requests)] - for package in reqs: if package.name not in BAD_PACKAGES: if package.link is not None: package_string = ( - '-e {0}'.format( + '-e {0}'.format(package.link) if package.editable else str( package.link - ) if package.editable else str(package.link) + ) ) project.add_package_to_pipfile(package_string, dev=dev) else: project.add_package_to_pipfile(str(package.req), dev=dev) - for index in indexes: project.add_index_to_pipfile(index) - project.recase_pipfile() @@ -264,8 +283,9 @@ def ensure_environment(): 'proper expected behavior.'.format( crayons.red('Warning', bold=True), crayons.normal('LANG', bold=True), - crayons.green('~/.profile') - ), err=True + crayons.green('~/.profile'), + ), + err=True, ) @@ -277,30 +297,30 @@ def import_from_code(path='.'): rs.append(r) pkg_names = pipreqs.get_pkg_names(rs) return [proper_case(r) for r in pkg_names] + except Exception: return [] def ensure_pipfile(validate=True, skip_requirements=False): """Creates a Pipfile for the project, if it doesn't exist.""" - global USING_DEFAULT_PYTHON - # Assert Pipfile exists. if project.pipfile_is_empty: - # If there's a requirements file, but no Pipfile... if project.requirements_exists and not skip_requirements: - click.echo(crayons.normal(u'requirements.txt found, instead of Pipfile! Convertingā€¦', bold=True)) - + click.echo( + crayons.normal( + u'requirements.txt found, instead of Pipfile! Convertingā€¦', + bold=True, + ) + ) # Create a Pipfile... python = which('python') if not USING_DEFAULT_PYTHON else None project.create_pipfile(python=python) - with spinner(): # Import requirements.txt. import_requirements() - # Warn the user of side-effects. click.echo( u'{0}: Your {1} now contains pinned versions, if your {2} did. \n' @@ -309,26 +329,27 @@ def ensure_pipfile(validate=True, skip_requirements=False): crayons.red('Warning', bold=True), crayons.normal('Pipfile', bold=True), crayons.normal('requirements.txt', bold=True), - crayons.normal('"*"', bold=True) + crayons.normal('"*"', bold=True), ) ) - else: - click.echo(crayons.normal(u'Creating a Pipfile for this projectā€¦', bold=True), err=True) - + click.echo( + crayons.normal(u'Creating a Pipfile for this projectā€¦', bold=True), + err=True, + ) # Create the pipfile if it doesn't exist. python = which('python') if not USING_DEFAULT_PYTHON else False project.create_pipfile(python=python) - # Validate the Pipfile's contents. if validate and project.virtualenv_exists and not PIPENV_SKIP_VALIDATION: # Ensure that Pipfile is using proper casing. p = project.parsed_pipfile changed = ensure_proper_casing(pfile=p) - # Write changes out to disk. if changed: - click.echo(crayons.normal(u'Fixing package names in Pipfileā€¦', bold=True), err=True) + click.echo( + crayons.normal(u'Fixing package names in Pipfileā€¦', bold=True), err=True + ) project.write_toml(p) @@ -344,13 +365,16 @@ def find_python_from_py(python): version_args = ['-{0}'.format(python[0])] if len(python) >= 2: version_args.append('-{0}.{1}'.format(python[0], python[2])) - import subprocess + for ver_arg in reversed(version_args): try: - python_exe = subprocess.check_output([py, ver_arg, '-c', 'import sys; print(sys.executable)']) + python_exe = subprocess.check_output( + [py, ver_arg, '-c', 'import sys; print(sys.executable)'] + ) except subprocess.CalledProcessError: continue + if not isinstance(python_exe, str): python_exe = python_exe.decode(sys.getdefaultencoding()) python_exe = python_exe.strip() @@ -364,29 +388,22 @@ def find_python_in_path(python): This uses the PATH environment variable to locate an appropriate Python. """ - possibilities = [ - 'python', - 'python{0}'.format(python[0]), - ] + possibilities = ['python', 'python{0}'.format(python[0])] if len(python) >= 2: possibilities.extend( [ 'python{0}{1}'.format(python[0], python[2]), 'python{0}.{1}'.format(python[0], python[2]), - 'python{0}.{1}m'.format(python[0], python[2]) + 'python{0}.{1}m'.format(python[0], python[2]), ] ) - # Reverse the list, so we find specific ones first. possibilities = reversed(possibilities) - for possibility in possibilities: # Windows compatibility. if os.name == 'nt': possibility = '{0}.exe'.format(possibility) - pythons = system_which(possibility, mult=True) - for p in pythons: version = python_version(p) if (version or '').startswith(python): @@ -397,16 +414,18 @@ def find_a_system_python(python): """Finds a system python, given a version (e.g. 2 / 2.7 / 3.6.2), or a full path.""" if python.startswith('py'): return system_which(python) + elif os.path.isabs(python): return python + python_from_py = find_python_from_py(python) if python_from_py: return python_from_py + return find_python_in_path(python) def ensure_python(three=None, python=None): - # Support for the PIPENV_PYTHON environment variable. if PIPENV_PYTHON and python is False and three is None: python = PIPENV_PYTHON @@ -414,60 +433,55 @@ def ensure_python(three=None, python=None): def abort(): click.echo( 'You can specify specific versions of Python with:\n {0}'.format( - crayons.red('$ pipenv --python {0}'.format(os.sep.join(('path', 'to', 'python')))) - ), err=True + crayons.red( + '$ pipenv --python {0}'.format( + os.sep.join(('path', 'to', 'python')) + ) + ) + ), + err=True, ) sys.exit(1) def activate_pyenv(): import pip9 + """Adds all pyenv installations to the PATH.""" if PYENV_INSTALLED: if PYENV_ROOT: pyenv_paths = {} - for found in glob( - '{0}{1}versions{1}*'.format( - PYENV_ROOT, - os.sep + for found in glob('{0}{1}versions{1}*'.format(PYENV_ROOT, os.sep)): + pyenv_paths[os.path.split(found)[1]] = '{0}{1}bin'.format( + found, os.sep ) - ): - pyenv_paths[os.path.split(found)[1]] = '{0}{1}bin'.format(found, os.sep) - for version_str, pyenv_path in pyenv_paths.items(): version = pip9._vendor.packaging.version.parse(version_str) if version.is_prerelease and pyenv_paths.get(version.base_version): continue + add_to_path(pyenv_path) else: click.echo( '{0}: PYENV_ROOT is not set. New python paths will ' 'probably not be exported properly after installation.' - ''.format( - crayons.red('Warning', bold=True), - ), err=True + ''.format(crayons.red('Warning', bold=True),), + err=True, ) global USING_DEFAULT_PYTHON - # Add pyenv paths to PATH. activate_pyenv() - path_to_python = None USING_DEFAULT_PYTHON = (three is None and not python) - # Find out which python is desired. if not python: python = convert_three_to_python(three, python) - if not python: python = project.required_python_version - if not python: python = PIPENV_DEFAULT_PYTHON_VERSION - if python: path_to_python = find_a_system_python(python) - if not path_to_python and python is not None: # We need to install Python. click.echo( @@ -475,7 +489,8 @@ def activate_pyenv(): crayons.red('Warning', bold=True), crayons.blue(python), u'was not found on your systemā€¦', - ), err=True + ), + err=True, ) # Pyenv is installed if not PYENV_INSTALLED: @@ -498,26 +513,22 @@ def activate_pyenv(): try: if len(python.split('.')) == 2: # Find the latest version of Python available. - version = version_map[python] else: version = python except KeyError: abort() - s = ( '{0} {1} {2}'.format( 'Would you like us to install', crayons.green('CPython {0}'.format(version)), - 'with pyenv?' + 'with pyenv?', ) ) - # Prompt the user to continue... if not (PIPENV_YES or click.confirm(s, default=True)): abort() else: - # Tell the user we're installing Python. click.echo( u'{0} {1} {2} {3}{4}'.format( @@ -525,36 +536,29 @@ def activate_pyenv(): crayons.green(u'CPython {0}'.format(version), bold=True), crayons.normal(u'with pyenv', bold=True), crayons.normal(u'(this may take a few minutes)'), - crayons.normal(u'ā€¦', bold=True) + crayons.normal(u'ā€¦', bold=True), ) ) - with spinner(): # Install Python. c = delegator.run( 'pyenv install {0} -s'.format(version), timeout=PIPENV_INSTALL_TIMEOUT, - block=False + block=False, ) - # Wait until the process has finished... c.block() - try: assert c.return_code == 0 except AssertionError: click.echo(u'Something went wrongā€¦') click.echo(crayons.blue(c.err), err=True) - # Print the results, in a beautiful blue... click.echo(crayons.blue(c.out), err=True) - # Add new paths to PATH. activate_pyenv() - # Find the newly installed Python, hopefully. path_to_python = find_a_system_python(version) - try: assert python_version(path_to_python) == version except AssertionError: @@ -562,11 +566,11 @@ def activate_pyenv(): '{0}: The Python you just installed is not available on your {1}, apparently.' ''.format( crayons.red('Warning', bold=True), - crayons.normal('PATH', bold=True) - ), err=True + crayons.normal('PATH', bold=True), + ), + err=True, ) sys.exit(1) - return path_to_python @@ -577,15 +581,12 @@ def abort(): sys.exit(1) global USING_DEFAULT_PYTHON - if not project.virtualenv_exists: try: # Ensure environment variables are set properly. ensure_environment() - # Ensure Python is available. python = ensure_python(three=three, python=python) - # Create the virtualenv. # Abort if --system (or running in a virtualenv). if PIPENV_USE_SYSTEM: @@ -597,57 +598,59 @@ def abort(): ) sys.exit(1) do_create_virtualenv(python=python, site_packages=site_packages) - except KeyboardInterrupt: # If interrupted, cleanup the virtualenv. cleanup_virtualenv(bare=False) sys.exit(1) - # If --three, --two, or --python were passed... elif (python) or (three is not None) or (site_packages is not False): - USING_DEFAULT_PYTHON = False - # Ensure python is installed before deleting existing virtual env ensure_python(three=three, python=python) - click.echo(crayons.red('Virtualenv already exists!'), err=True) # If VIRTUAL_ENV is set, there is a possibility that we are # going to remove the active virtualenv that the user cares # about, so confirm first. if 'VIRTUAL_ENV' in os.environ: - if not (PIPENV_YES or click.confirm('Remove existing virtualenv?', default=True)): + if not ( + PIPENV_YES or click.confirm('Remove existing virtualenv?', default=True) + ): abort() - click.echo(crayons.normal(u'Removing existing virtualenvā€¦', bold=True), err=True) - + click.echo( + crayons.normal(u'Removing existing virtualenvā€¦', bold=True), err=True + ) # Remove the virtualenv. cleanup_virtualenv(bare=True) - # Call this function again. ensure_virtualenv(three=three, python=python, site_packages=site_packages) -def ensure_project(three=None, python=None, validate=True, system=False, warn=True, site_packages=False, deploy=False, skip_requirements=False): +def ensure_project( + three=None, + python=None, + validate=True, + system=False, + warn=True, + site_packages=False, + deploy=False, + skip_requirements=False, +): """Ensures both Pipfile and virtualenv exist for the project.""" - # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - if not project.pipfile_exists: project.touch_pipfile() - # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv(three=three, python=python, site_packages=site_packages) - if warn: # Warn users if they are using the wrong version of Python. if project.required_python_version: - path_to_python = which('python') or which('py') - - if path_to_python and project.required_python_version not in (python_version(path_to_python) or ''): + if path_to_python and project.required_python_version not in ( + python_version(path_to_python) or '' + ): click.echo( '{0}: Your Pipfile requires {1} {2}, ' 'but you are using {3} ({4}).'.format( @@ -655,29 +658,27 @@ def ensure_project(three=None, python=None, validate=True, system=False, warn=Tr crayons.normal('python_version', bold=True), crayons.blue(project.required_python_version), crayons.blue(python_version(path_to_python)), - crayons.green(shorten_path(path_to_python)) - ), err=True + crayons.green(shorten_path(path_to_python)), + ), + err=True, ) if not deploy: click.echo( ' {0} will surely fail.' ''.format(crayons.red('$ pipenv check')), - err=True + err=True, ) else: click.echo(crayons.red('Deploy aborted.'), err=True) sys.exit(1) - # Ensure the Pipfile exists. ensure_pipfile(validate=validate, skip_requirements=skip_requirements) def ensure_proper_casing(pfile): """Ensures proper casing of Pipfile packages, writes changes to disk.""" - casing_changed = proper_case_section(pfile.get('packages', {})) casing_changed |= proper_case_section(pfile.get('dev-packages', {})) - return casing_changed @@ -685,11 +686,9 @@ def proper_case_section(section): """Verify proper casing is retrieved, when available, for each dependency in the section. """ - # Casing for section. changed_values = False unknown_names = [k for k in section.keys() if k not in set(project.proper_names)] - # Replace each package with proper casing. for dep in unknown_names: try: @@ -702,12 +701,10 @@ def proper_case_section(section): if new_casing != dep: changed_values = True project.register_proper_name(new_casing) - # Replace old value with new value. old_value = section[dep] section[new_casing] = old_value del section[dep] - # Return whether or not values have been changed. return changed_values @@ -715,51 +712,63 @@ def proper_case_section(section): def shorten_path(location, bold=False): """Returns a visually shorter representation of a given system path.""" original = location - short = os.sep.join([s[0] if len(s) > (len('2long4')) else s for s in location.split(os.sep)]) + short = os.sep.join( + [s[0] if len(s) > (len('2long4')) else s for s in location.split(os.sep)] + ) short = short.split(os.sep) short[-1] = original.split(os.sep)[-1] if bold: short[-1] = str(crayons.normal(short[-1], bold=True)) - return os.sep.join(short) - # return short + + +# return short def do_where(virtualenv=False, bare=True): """Executes the where functionality.""" - if not virtualenv: location = project.pipfile_location - # Shorten the virtual display of the path to the virtualenv. if not bare: location = shorten_path(location) - if not location: click.echo( 'No Pipfile present at project home. Consider running ' '{0} first to automatically generate a Pipfile for you.' - ''.format(crayons.green('`pipenv install`')), err=True) + ''.format(crayons.green('`pipenv install`')), + err=True, + ) elif not bare: click.echo( 'Pipfile found at {0}.\n Considering this to be the project home.' - ''.format(crayons.green(location)), err=True) + ''.format(crayons.green(location)), + err=True, + ) pass else: click.echo(project.project_directory) - else: location = project.virtualenv_location - if not bare: - click.echo('Virtualenv location: {0}'.format(crayons.green(location)), err=True) + click.echo( + 'Virtualenv location: {0}'.format(crayons.green(location)), err=True + ) else: click.echo(location) def do_install_dependencies( - dev=False, only=False, bare=False, requirements=False, allow_global=False, - ignore_hashes=False, skip_lock=False, verbose=False, concurrent=True, requirements_dir=None + dev=False, + only=False, + bare=False, + requirements=False, + allow_global=False, + ignore_hashes=False, + skip_lock=False, + verbose=False, + concurrent=True, + requirements_dir=None, ): """"Executes the install functionality. @@ -768,57 +777,48 @@ def do_install_dependencies( def cleanup_procs(procs, concurrent): for c in procs: - if concurrent: c.block() - if 'Ignoring' in c.out: click.echo(crayons.yellow(c.out.strip())) - if verbose: click.echo(crayons.blue(c.out or c.err)) - # The Installation failed... if c.return_code != 0: - # Save the Failed Dependency for later. failed_deps_list.append((c.dep, c.ignore_hash)) - # Alert the user. click.echo( '{0} {1}! Will try again.'.format( crayons.red('An error occurred while installing'), - crayons.green(c.dep.split('--hash')[0].strip()) + crayons.green(c.dep.split('--hash')[0].strip()), ) ) if requirements: bare = True - blocking = (not concurrent) - # Load the lockfile if it exists, or if only is being used (e.g. lock is being used). if skip_lock or only or not project.lockfile_exists: if not bare: - click.echo(crayons.normal(u'Installing dependencies from Pipfileā€¦', bold=True)) + click.echo( + crayons.normal(u'Installing dependencies from Pipfileā€¦', bold=True) + ) lockfile = split_file(project._lockfile) else: with open(project.lockfile_location) as f: lockfile = split_file(simplejson.load(f)) - if not bare: click.echo( crayons.normal( u'Installing dependencies from Pipfile.lock ({0})ā€¦'.format( lockfile['_meta'].get('hash', {}).get('sha256')[-6:] ), - bold=True + bold=True, ) ) - # Allow pip to resolve dependencies when in skip-lock mode. no_deps = (not skip_lock) - deps_list, dev_deps_list = merge_deps( lockfile, project, @@ -826,36 +826,30 @@ def cleanup_procs(procs, concurrent): requirements=requirements, ignore_hashes=ignore_hashes, blocking=blocking, - only=only + only=only, ) failed_deps_list = [] - if requirements: - # Comment out packages that shouldn't be included in # requirements.txt, for pip9. - # Additional package selectors, specific to pip's --hash checking mode. for l in (deps_list, dev_deps_list): for i, dep in enumerate(l): l[i] = list(l[i]) if '--hash' in l[i][0]: l[i][0] = (l[i][0].split('--hash')[0].strip()) - # Output only default dependencies if not dev: click.echo('\n'.join(d[0] for d in sorted(deps_list))) sys.exit(0) - # Output only dev dependencies if dev: click.echo('\n'.join(d[0] for d in sorted(dev_deps_list))) sys.exit(0) - procs = [] - - deps_list_bar = progress.bar(deps_list, label=INSTALL_LABEL if os.name != 'nt' else '') - + deps_list_bar = progress.bar( + deps_list, label=INSTALL_LABEL if os.name != 'nt' else '' + ) for dep, ignore_hash, block in deps_list_bar: if len(procs) < PIPENV_MAX_SUBPROCESS: # Use a specific index, if specified. @@ -864,7 +858,6 @@ def cleanup_procs(procs, concurrent): dep, index = dep.split(' -i ') dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip() index = index.split()[0] - # Install the module. c = pip_install( dep, @@ -874,32 +867,26 @@ def cleanup_procs(procs, concurrent): verbose=verbose, block=block, index=index, - requirements_dir=requirements_dir + requirements_dir=requirements_dir, ) - c.dep = dep c.ignore_hash = ignore_hash - procs.append(c) - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): cleanup_procs(procs, concurrent) procs = [] - cleanup_procs(procs, concurrent) - # Iterate over the hopefully-poorly-packaged dependencies... if failed_deps_list: - - click.echo(crayons.normal(u'Installing initiallyā€“failed dependenciesā€¦', bold=True)) - + click.echo( + crayons.normal(u'Installing initiallyā€“failed dependenciesā€¦', bold=True) + ) for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): index = None if ' -i ' in dep: dep, index = dep.split(' -i ') dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip() index = index.split()[0] - # Install the module. c = pip_install( dep, @@ -908,70 +895,71 @@ def cleanup_procs(procs, concurrent): no_deps=no_deps, verbose=verbose, index=index, - requirements_dir=requirements_dir + requirements_dir=requirements_dir, ) - # The Installation failed... if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. click.echo(crayons.blue(format_pip_output(c.out))) click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. sys.exit(c.return_code) else: - click.echo('{0} {1}{2}'.format( - crayons.green('Success installing'), - crayons.green(dep.split('--hash')[0].strip()), - crayons.green('!') - )) + click.echo( + '{0} {1}{2}'.format( + crayons.green('Success installing'), + crayons.green(dep.split('--hash')[0].strip()), + crayons.green('!'), + ) + ) def convert_three_to_python(three, python): """Converts a Three flag into a Python flag, and raises customer warnings in the process, if needed. """ - if not python: if three is False: return '2' elif three is True: return '3' + else: return python def do_create_virtualenv(python=None, site_packages=False): """Creates a virtualenv.""" - - click.echo(crayons.normal(u'Creating a virtualenv for this projectā€¦', bold=True), err=True) - + click.echo( + crayons.normal(u'Creating a virtualenv for this projectā€¦', bold=True), err=True + ) # The user wants the virtualenv in the project. if PIPENV_VENV_IN_PROJECT: - cmd = ['virtualenv', project.virtualenv_location, '--prompt=({0})'.format(project.name)] - + cmd = [ + 'virtualenv', + project.virtualenv_location, + '--prompt=({0})'.format(project.name), + ] # Pass site-packages flag to virtualenv, if desired... if site_packages: cmd.append('--system-site-packages') else: # Default: use pew. cmd = [sys.executable, '-m', 'pipenv.pew', 'new', project.virtualenv_name, '-d'] - # Default to using sys.executable, if Python wasn't provided. if not python: python = sys.executable - - click.echo(u'{0} {1} {3} {2}'.format( - crayons.normal('Using', bold=True), - crayons.red(python, bold=True), - crayons.normal(u'to create virtualenvā€¦', bold=True), - crayons.green('({0})'.format(python_version(python))) - ), err=True) - + click.echo( + u'{0} {1} {3} {2}'.format( + crayons.normal('Using', bold=True), + crayons.red(python, bold=True), + crayons.normal(u'to create virtualenvā€¦', bold=True), + crayons.green('({0})'.format(python_version(python))), + ), + err=True, + ) cmd = cmd + ['-p', python] - # Actually create the virtualenv. with spinner(): try: @@ -983,149 +971,124 @@ def do_create_virtualenv(python=None, site_packages=False): ''.format( crayons.red('Warning', bold=True), crayons.red(cmd[0]), - crayons.normal('PATH', bold=True) - ), err=True + crayons.normal('PATH', bold=True), + ), + err=True, ) sys.exit(1) - click.echo(crayons.blue(c.out), err=True) - # Enable site-packages, if desired... if not PIPENV_VENV_IN_PROJECT and site_packages: - - click.echo(crayons.normal(u'Making site-packages availableā€¦', bold=True), err=True) - + click.echo( + crayons.normal(u'Making site-packages availableā€¦', bold=True), err=True + ) os.environ['VIRTUAL_ENV'] = project.virtualenv_location delegator.run('pipenv run pewtwo toggleglobalsitepackages') del os.environ['VIRTUAL_ENV'] - # Say where the virtualenv is. do_where(virtualenv=True, bare=False) def parse_download_fname(fname, name): fname, fextension = os.path.splitext(fname) - if fextension == '.whl': fname = '-'.join(fname.split('-')[:-3]) - if fname.endswith('.tar'): fname, _ = os.path.splitext(fname) - # Substring out package name (plus dash) from file name to get version. version = fname[len(name) + 1:] - # Ignore implicit post releases in version number. if '-' in version and version.split('-')[1].isdigit(): version = version.split('-')[0] - return version def get_downloads_info(names_map, section): info = [] - p = project.parsed_pipfile - for fname in os.listdir(project.download_location): # Get name from filename mapping. name = list(convert_deps_from_pip(names_map[fname]))[0] # Get the version info from the filenames. version = parse_download_fname(fname, name) - # Get the hash of each file. cmd = '{0} hash "{1}"'.format( escape_grouped_arguments(which_pip()), - os.sep.join([project.download_location, fname]) + os.sep.join([project.download_location, fname]), ) - c = delegator.run(cmd) hash = c.out.split('--hash=')[1].strip() - # Verify we're adding the correct version from Pipfile # and not one from a dependency. specified_version = p[section].get(name, '') if is_required_version(version, specified_version): info.append(dict(name=name, version=version, hash=hash)) - return info -def do_lock(verbose=False, system=False, clear=False, pre=False, keep_outdated=False, write=True): +def do_lock( + verbose=False, system=False, clear=False, pre=False, keep_outdated=False, write=True +): """Executes the freeze functionality.""" - cached_lockfile = {} if keep_outdated: if not project.lockfile_exists: - click.echo('{0}: Pipfile.lock must exist to use --keep-outdated!'.format( - crayons.red('Warning', bold=True) - )) + click.echo( + '{0}: Pipfile.lock must exist to use --keep-outdated!'.format( + crayons.red('Warning', bold=True) + ) + ) sys.exit(1) cached_lockfile = project.lockfile_content - if write: project.destroy_lockfile() - if write: # Alert the user of progress. click.echo( u'{0} {1} {2}'.format( crayons.normal('Locking'), crayons.red('[dev-packages]'), - crayons.normal('dependenciesā€¦') + crayons.normal('dependenciesā€¦'), ), - err=True + err=True, ) - # Create the lockfile. lockfile = project._lockfile - # Cleanup lockfile. for section in ('default', 'develop'): for k, v in lockfile[section].copy().items(): if not hasattr(v, 'keys'): del lockfile[section][k] - # Ensure that develop inherits from default. dev_packages = project.dev_packages.copy() - for dev_package in project.dev_packages: if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] - # Resolve dev-package dependencies, with pip-tools. deps = convert_deps_to_pip(dev_packages, project, r=False, include_index=True) - results = venv_resolve_deps( - deps, - which=which, - verbose=verbose, - project=project, - clear=clear, - pre=pre, + deps, which=which, verbose=verbose, project=project, clear=clear, pre=pre ) - # Add develop dependencies to lockfile. for dep in results: # Add version information to lockfile. - lockfile['develop'].update({dep['name']: {'version': '=={0}'.format(dep['version'])}}) - + lockfile['develop'].update( + {dep['name']: {'version': '=={0}'.format(dep['version'])}} + ) # Add Hashes to lockfile lockfile['develop'][dep['name']]['hashes'] = sorted(dep['hashes']) - # Add index metadata to lockfile. if 'index' in dep: lockfile['develop'][dep['name']]['index'] = dep['index'] - # Add PEP 508 specifier metadata to lockfile. if 'markers' in dep: lockfile['develop'][dep['name']]['markers'] = dep['markers'] - # Add refs for VCS installs. # TODO: be smarter about this. vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False) - pip_freeze = delegator.run('{0} freeze'.format(escape_grouped_arguments(which_pip()))).out - + pip_freeze = delegator.run( + '{0} freeze'.format(escape_grouped_arguments(which_pip())) + ).out if vcs_deps: for line in pip_freeze.strip().split('\n'): # if the line doesn't match a vcs dependency in the Pipfile, @@ -1136,101 +1099,87 @@ def do_lock(verbose=False, system=False, clear=False, pre=False, keep_outdated=F try: installed = convert_deps_from_pip(line) name = list(installed.keys())[0] - if is_vcs(installed[name]): lockfile['develop'].update(installed) - except IndexError: pass - if write: # Alert the user of progress. click.echo( u'{0} {1} {2}'.format( crayons.normal('Locking'), crayons.red('[packages]'), - crayons.normal('dependenciesā€¦') + crayons.normal('dependenciesā€¦'), ), - err=True + err=True, ) - # Resolve package dependencies, with pip-tools. deps = convert_deps_to_pip(project.packages, project, r=False, include_index=True) results = venv_resolve_deps( - deps, - which=which, - verbose=verbose, - project=project, - clear=clear, - pre=pre, + deps, which=which, verbose=verbose, project=project, clear=clear, pre=pre ) - # Add default dependencies to lockfile. for dep in results: # Add version information to lockfile. - lockfile['default'].update({dep['name']: {'version': '=={0}'.format(dep['version'])}}) - + lockfile['default'].update( + {dep['name']: {'version': '=={0}'.format(dep['version'])}} + ) # Add Hashes to lockfile lockfile['default'][dep['name']]['hashes'] = sorted(dep['hashes']) - # Add index metadata to lockfile. if 'index' in dep: lockfile['default'][dep['name']]['index'] = dep['index'] - # Add PEP 508 specifier metadata to lockfile. if 'markers' in dep: lockfile['default'][dep['name']]['markers'] = dep['markers'] - # Add refs for VCS installs. # TODO: be smarter about this. vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False) pip_freeze = delegator.run('{0} freeze'.format(which_pip())).out - for dep in vcs_deps: for line in pip_freeze.strip().split('\n'): try: installed = convert_deps_from_pip(line) name = list(installed.keys())[0] - if is_vcs(installed[name]): # Convert name to PEP 423 name. installed = {pep423_name(name): installed[name]} - lockfile['default'].update(installed) except IndexError: pass - # Support for --keep-outdatedā€¦ if keep_outdated: - for section_name, section in (('default', project.packages), ('develop', project.dev_packages)): + for section_name, section in ( + ('default', project.packages), ('develop', project.dev_packages) + ): for package_specified in section: norm_name = pep423_name(package_specified) if not is_pinned(section[package_specified]): - lockfile[section_name][norm_name] = cached_lockfile[section_name][norm_name] - + lockfile[section_name][norm_name] = cached_lockfile[section_name][ + norm_name + ] # Overwrite any develop packages with default packages. for default_package in lockfile['default']: if default_package in lockfile['develop']: lockfile['develop'][default_package] = lockfile['default'][default_package] - if write: - # Write out the lockfile. with open(project.lockfile_location, 'w') as f: - simplejson.dump(lockfile, f, indent=4, separators=(',', ': '), sort_keys=True) + simplejson.dump( + lockfile, f, indent=4, separators=(',', ': '), sort_keys=True + ) # Write newline at end of document. GH Issue #319. f.write('\n') - click.echo( '{0}'.format( crayons.normal( 'Updated Pipfile.lock ({0})!'.format( lockfile['_meta'].get('hash', {}).get('sha256')[-6:] ), - bold=True + bold=True, ) ), - err=True + err=True, ) else: return lockfile @@ -1238,27 +1187,23 @@ def do_lock(verbose=False, system=False, clear=False, pre=False, keep_outdated=F def activate_virtualenv(source=True): """Returns the string to activate a virtualenv.""" - # Suffix and source command for other shells. suffix = '' command = '.' if source else '' - # Support for fish shell. if PIPENV_SHELL and 'fish' in PIPENV_SHELL: suffix = '.fish' command = 'source' - # Support for csh shell. if PIPENV_SHELL and 'csh' in PIPENV_SHELL: suffix = '.csh' command = 'source' - # Escape any spaces located within the virtualenv path to allow # for proper activation. venv_location = project.virtualenv_location.replace(' ', r'\ ') - if source: return '{2} {0}/bin/activate{1}'.format(venv_location, suffix, command) + else: return '{0}/bin/activate'.format(venv_location) @@ -1268,8 +1213,10 @@ def do_activate_virtualenv(bare=False): # Check for environment marker, and skip if it's set. if 'PIPENV_ACTIVE' not in os.environ: if not bare: - click.echo('To activate this project\'s virtualenv, run the following:\n $ {0}'.format( - crayons.red('pipenv shell')) + click.echo( + 'To activate this project\'s virtualenv, run the following:\n $ {0}'.format( + crayons.red('pipenv shell') + ) ) else: click.echo(activate_virtualenv()) @@ -1277,60 +1224,68 @@ def do_activate_virtualenv(bare=False): def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): """Executes the purge functionality.""" - if downloads: if not bare: click.echo(crayons.normal(u'Clearing out downloads directoryā€¦', bold=True)) shutil.rmtree(project.download_location) return - freeze = delegator.run('{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=allow_global)))).out - + freeze = delegator.run( + '{0} freeze'.format( + escape_grouped_arguments(which_pip(allow_global=allow_global)) + ) + ).out # Remove comments from the output, if any. - installed = [line for line in freeze.splitlines() if not line.lstrip().startswith('#')] - + installed = [ + line for line in freeze.splitlines() if not line.lstrip().startswith('#') + ] # Remove setuptools and friends from installed, if present. for package_name in BAD_PACKAGES: for i, package in enumerate(installed): if package.startswith(package_name): del installed[i] - actually_installed = [] - for package in installed: try: dep = convert_deps_from_pip(package) except AssertionError: dep = None - if dep and not is_vcs(dep): - dep = [k for k in dep.keys()][0] # TODO: make this smarter later. if not dep.startswith('-e ') and not dep.startswith('git+'): actually_installed.append(dep) - if not bare: - click.echo(u'Found {0} installed package(s), purgingā€¦'.format(len(actually_installed))) - command = '{0} uninstall {1} -y'.format(escape_grouped_arguments(which_pip(allow_global=allow_global)), ' '.join(actually_installed)) - + click.echo( + u'Found {0} installed package(s), purgingā€¦'.format(len(actually_installed)) + ) + command = '{0} uninstall {1} -y'.format( + escape_grouped_arguments(which_pip(allow_global=allow_global)), + ' '.join(actually_installed), + ) if verbose: click.echo('$ {0}'.format(command)) - c = delegator.run(command) - if not bare: click.echo(crayons.blue(c.out)) click.echo(crayons.green('Environment now purged and fresh!')) def do_init( - dev=False, requirements=False, allow_global=False, ignore_pipfile=False, - skip_lock=False, verbose=False, system=False, concurrent=True, deploy=False, - pre=False, keep_outdated=False, requirements_dir=None + dev=False, + requirements=False, + allow_global=False, + ignore_pipfile=False, + skip_lock=False, + verbose=False, + system=False, + concurrent=True, + deploy=False, + pre=False, + keep_outdated=False, + requirements_dir=None, ): """Executes the init functionality.""" - if not system: if not project.virtualenv_exists: try: @@ -1338,34 +1293,26 @@ def do_init( except KeyboardInterrupt: cleanup_virtualenv(bare=False) sys.exit(1) - # Ensure the Pipfile exists. ensure_pipfile() - if not requirements_dir: requirements_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') - # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored if (project.lockfile_exists and not ignore_pipfile) and not skip_lock: - # Open the lockfile. with codecs.open(project.lockfile_location, 'r') as f: lockfile = simplejson.load(f) - # Update the lockfile if it is out-of-date. p = pipfile.load(project.pipfile_location) - # Check that the hash of the Lockfile matches the lockfile's hash. if not lockfile['_meta'].get('hash', {}).get('sha256') == p.hash: - old_hash = lockfile['_meta'].get('hash', {}).get('sha256')[-6:] new_hash = p.hash[-6:] if deploy: click.echo( crayons.red( 'Your Pipfile.lock ({0}) is out of date. Expected: ({1}).'.format( - old_hash, - new_hash + old_hash, new_hash ) ) ) @@ -1376,56 +1323,73 @@ def do_init( click.echo( crayons.red( u'Pipfile.lock ({0}) out of date, updating to ({1})ā€¦'.format( - old_hash, - new_hash + old_hash, new_hash ), - bold=True), - err=True + bold=True, + ), + err=True, ) - do_lock(system=system, pre=pre) - # Write out the lockfile if it doesn't exist. if not project.lockfile_exists and not skip_lock: - click.echo(crayons.normal(u'Pipfile.lock not found, creatingā€¦', bold=True), err=True) + click.echo( + crayons.normal(u'Pipfile.lock not found, creatingā€¦', bold=True), err=True + ) do_lock(system=system, pre=pre, keep_outdated=keep_outdated, verbose=verbose) - - do_install_dependencies(dev=dev, requirements=requirements, allow_global=allow_global, - skip_lock=skip_lock, verbose=verbose, concurrent=concurrent, - requirements_dir=requirements_dir.name) + do_install_dependencies( + dev=dev, + requirements=requirements, + allow_global=allow_global, + skip_lock=skip_lock, + verbose=verbose, + concurrent=concurrent, + requirements_dir=requirements_dir.name, + ) requirements_dir.cleanup() - # Activate virtualenv instructions. if not allow_global and not deploy: do_activate_virtualenv() def pip_install( - package_name=None, r=None, allow_global=False, ignore_hashes=False, - no_deps=True, verbose=False, block=True, index=None, pre=False, - selective_upgrade=False, requirements_dir=None + package_name=None, + r=None, + allow_global=False, + ignore_hashes=False, + no_deps=True, + verbose=False, + block=True, + index=None, + pre=False, + selective_upgrade=False, + requirements_dir=None, ): import pip9 if verbose: - click.echo(crayons.normal('Installing {0!r}'.format(package_name), bold=True), err=True) + click.echo( + crayons.normal('Installing {0!r}'.format(package_name), bold=True), err=True + ) pip9.logger.setLevel(logging.INFO) - # Create files for hash mode. if not package_name.startswith('-e ') and (not ignore_hashes) and (r is None): - fd, r = tempfile.mkstemp(prefix='pipenv-', suffix='-requirement.txt', dir=requirements_dir) + fd, r = tempfile.mkstemp( + prefix='pipenv-', suffix='-requirement.txt', dir=requirements_dir + ) with os.fdopen(fd, 'w') as f: f.write(package_name) - # Install dependencies when a package is a VCS dependency. try: - req = get_requirement(package_name.split('--hash')[0].split('--trusted-host')[0]).vcs + req = get_requirement( + package_name.split('--hash')[0].split('--trusted-host')[0] + ).vcs except (pip9._vendor.pyparsing.ParseException, ValueError) as e: click.echo('{0}: {1}'.format(crayons.red('WARNING'), e), err=True) click.echo( '{0}... You will have to reinstall any packages that failed to install.'.format( crayons.red('ABORTING INSTALL') - ), err=True + ), + err=True, ) click.echo( 'You may have to manually run {0} when you are finished.'.format( @@ -1435,21 +1399,20 @@ def pip_install( sys.exit(1) if req: no_deps = False - # Don't specify a source directory when using --system. if not allow_global and ('PIP_SRC' not in os.environ): - src = '--src {0}'.format(escape_grouped_arguments(project.virtualenv_src_location)) + src = '--src {0}'.format( + escape_grouped_arguments(project.virtualenv_src_location) + ) else: src = '' else: src = '' - # Try installing for each source in project.sources. if index: sources = [{'url': index}] else: sources = project.sources - for source in sources: if package_name.startswith('-e '): install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1]) @@ -1457,7 +1420,6 @@ def pip_install( install_reqs = ' -r {0}'.format(r) else: install_reqs = ' "{0}"'.format(package_name) - # Skip hash-checking mode, when appropriate. if r: with open(r) as f: @@ -1466,19 +1428,14 @@ def pip_install( else: if '--hash' not in install_reqs: ignore_hashes = True - verbose_flag = '--verbose' if verbose else '' - if not ignore_hashes: install_reqs += ' --require-hashes' - no_deps = '--no-deps' if no_deps else '' pre = '--pre' if pre else '' - quoted_pip = which_pip(allow_global=allow_global) quoted_pip = escape_grouped_arguments(quoted_pip) upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' - pip_command = '{0} install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( quoted_pip, install_reqs, @@ -1487,12 +1444,10 @@ def pip_install( pre, src, verbose_flag, - upgrade_strategy + upgrade_strategy, ) - if verbose: click.echo('$ {0}'.format(pip_command), err=True) - c = delegator.run(pip_command, block=block) if c.return_code == 0: break @@ -1507,11 +1462,12 @@ def pip_download(package_name): delegator.run(which_pip()), package_name, source['url'], - project.download_location + project.download_location, ) c = delegator.run(cmd) if c.return_code == 0: break + return c @@ -1531,9 +1487,7 @@ def which_pip(allow_global=False): def system_which(command, mult=False): """Emulates the system's which. Returns None if not found.""" - _which = 'which -a' if not os.name == 'nt' else 'where' - c = delegator.run('{0} {1}'.format(_which, command)) try: # Which Not found... @@ -1541,18 +1495,18 @@ def system_which(command, mult=False): click.echo( '{}: the {} system utility is required for Pipenv to find Python installations properly.' '\n Please install it.'.format( - crayons.red('Warning', bold=True), - crayons.red(_which) - ), err=True + crayons.red('Warning', bold=True), crayons.red(_which) + ), + err=True, ) assert c.return_code == 0 except AssertionError: return None if not mult else [] result = c.out.strip() or c.err.strip() - if mult: return result.split('\n') + else: return result.split('\n')[0] @@ -1560,9 +1514,9 @@ def system_which(command, mult=False): def format_help(help): """Formats the help string.""" help = help.replace('Options:', str(crayons.normal('Options:', bold=True))) - - help = help.replace('Usage: pipenv', str('Usage: {0}'.format(crayons.normal('pipenv', bold=True)))) - + help = help.replace( + 'Usage: pipenv', str('Usage: {0}'.format(crayons.normal('pipenv', bold=True))) + ) help = help.replace(' check', str(crayons.red(' check', bold=True))) help = help.replace(' clean', str(crayons.red(' clean', bold=True))) help = help.replace(' graph', str(crayons.red(' graph', bold=True))) @@ -1574,7 +1528,6 @@ def format_help(help): help = help.replace(' sync', str(crayons.green(' sync', bold=True))) help = help.replace(' uninstall', str(crayons.magenta(' uninstall', bold=True))) help = help.replace(' update', str(crayons.green(' update', bold=True))) - additional_help = """ Usage Examples: Create a new project using Python 3.6, specifically: @@ -1609,28 +1562,37 @@ def format_help(help): crayons.red('pipenv check'), crayons.red('pipenv run pip freeze'), ) - help = help.replace('Commands:', additional_help) - return help def format_pip_error(error): error = error.replace('Expected', str(crayons.green('Expected', bold=True))) error = error.replace('Got', str(crayons.red('Got', bold=True))) - error = error.replace('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE', str(crayons.red('THESE PACKAGES DO NOT MATCH THE HASHES FROM Pipfile.lock!', bold=True))) - error = error.replace('someone may have tampered with them', str(crayons.red('someone may have tampered with them'))) - + error = error.replace( + 'THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE', + str( + crayons.red( + 'THESE PACKAGES DO NOT MATCH THE HASHES FROM Pipfile.lock!', bold=True + ) + ), + ) + error = error.replace( + 'someone may have tampered with them', + str(crayons.red('someone may have tampered with them')), + ) error = error.replace('option to pip install', 'option to \'pipenv install\'') return error def format_pip_output(out, r=None): + def gen(out): for line in out.split('\n'): # Remove requirements file information from pip9 output. if '(from -r' in line: - yield line[:line.index('(from -r')] + yield line[: line.index('(from -r')] + else: yield line @@ -1638,11 +1600,12 @@ def gen(out): return out + + # |\/| /\ |) [- ]3 `/ # . . .-. . . . . .-. .-. . . .-. .-. .-. .-. .-. # |< |- |\| |\| |- | |-| |( |- | | / # ' ` `-' ' ` ' ` `-' ' ' ` ' ' `-' `-' ' `-' - def warn_in_virtualenv(): if PIPENV_USE_SYSTEM: # Only warn if pipenv isn't already active. @@ -1652,7 +1615,8 @@ def warn_in_virtualenv(): 'so it will automatically use that environment, instead of ' 'creating its own for any project.'.format( crayons.green('Courtesy Notice') - ), err=True + ), + err=True, ) @@ -1661,33 +1625,26 @@ def ensure_lockfile(keep_outdated=False): pre = project.settings.get('allow_prereleases') if not keep_outdated: keep_outdated = project.settings.get('keep_outdated') - # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored if project.lockfile_exists: - # Open the lockfile. with codecs.open(project.lockfile_location, 'r') as f: lockfile = simplejson.load(f) - # Update the lockfile if it is out-of-date. p = pipfile.load(project.pipfile_location) - # Check that the hash of the Lockfile matches the lockfile's hash. if not lockfile['_meta'].get('hash', {}).get('sha256') == p.hash: - old_hash = lockfile['_meta'].get('hash', {}).get('sha256')[-6:] new_hash = p.hash[-6:] - click.echo( crayons.red( u'Pipfile.lock ({0}) out of date, updating to ({1})ā€¦'.format( - old_hash, - new_hash + old_hash, new_hash ), - bold=True), - err=True + bold=True, + ), + err=True, ) - do_lock(pre=pre, keep_outdated=keep_outdated) else: do_lock(pre=pre, keep_outdated=keep_outdated) @@ -1699,15 +1656,14 @@ def do_py(system=False): except AttributeError: click.echo(crayons.red('No project found!')) + def do_outdated(): packages = {} results = delegator.run('{0} freeze'.format(which('pip'))).out.strip().split('\n') results = filter(bool, results) for result in results: packages.update(convert_deps_from_pip(result)) - updated_packages = {} - lockfile = do_lock(write=False) for section in ('develop', 'default'): for package in lockfile[section]: @@ -1715,171 +1671,204 @@ def do_outdated(): updated_packages[package] = lockfile[section][package]['version'] except KeyError: pass - outdated = [] for package in packages: if package in updated_packages: if updated_packages[package] != packages[package]: outdated.append((package, updated_packages[package], packages[package])) - for package, new_version, old_version in outdated: - click.echo('Package {0!r} outā€“ofā€“date: {1!r} installed, {2!r} available.'.format(package, old_version, new_version)) - + click.echo( + 'Package {0!r} outā€“ofā€“date: {1!r} installed, {2!r} available.'.format( + package, old_version, new_version + ) + ) sys.exit(bool(outdated)) - def do_install( - package_name=False, more_packages=False, dev=False, three=False, - python=False, system=False, lock=True, ignore_pipfile=False, - skip_lock=False, verbose=False, requirements=False, sequential=False, - pre=False, code=False, deploy=False, keep_outdated=False, - selective_upgrade=False + package_name=False, + more_packages=False, + dev=False, + three=False, + python=False, + system=False, + lock=True, + ignore_pipfile=False, + skip_lock=False, + verbose=False, + requirements=False, + sequential=False, + pre=False, + code=False, + deploy=False, + keep_outdated=False, + selective_upgrade=False, ): import pip9 - requirements_directory = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') + requirements_directory = TemporaryDirectory( + suffix='-requirements', prefix='pipenv-' + ) if selective_upgrade: keep_outdated = True - more_packages = more_packages or [] - # Don't search for requirements.txt files if the user provides one skip_requirements = True if requirements else False - concurrent = (not sequential) - # Ensure that virtualenv is available. - ensure_project(three=three, python=python, system=system, warn=True, deploy=deploy, skip_requirements=skip_requirements) - + ensure_project( + three=three, + python=python, + system=system, + warn=True, + deploy=deploy, + skip_requirements=skip_requirements, + ) # Load the --pre settings from the Pipfile. if not pre: pre = project.settings.get('allow_prereleases') - if not keep_outdated: keep_outdated = project.settings.get('keep_outdated') - remote = requirements and is_valid_url(requirements) - # Warn and exit if --system is used without a pipfile. if system and package_name: click.echo( '{0}: --system is intended to be used for Pipfile installation, ' 'not installation of specific packages. Aborting.'.format( crayons.red('Warning', bold=True) - ), err=True + ), + err=True, ) click.echo('See also: --deploy flag.', err=True) requirements_directory.cleanup() sys.exit(1) - # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp(prefix='pipenv-', suffix='-requirement.txt', dir=requirements_directory.name) + fd, temp_reqs = tempfile.mkstemp( + prefix='pipenv-', suffix='-requirement.txt', dir=requirements_directory.name + ) requirements_url = requirements - # Download requirements file - click.echo(crayons.normal(u'Remote requirements file provided! Downloadingā€¦', bold=True), err=True) + click.echo( + crayons.normal( + u'Remote requirements file provided! Downloadingā€¦', bold=True + ), + err=True, + ) try: download_file(requirements, temp_reqs) except IOError: click.echo( crayons.red( - u'Unable to find requirements file at {0}.'.format(crayons.normal(requirements)) + u'Unable to find requirements file at {0}.'.format( + crayons.normal(requirements) + ) ), - err=True + err=True, ) requirements_directory.cleanup() sys.exit(1) # Replace the url with the temporary requirements file requirements = temp_reqs remote = True - if requirements: error, traceback = None, None - click.echo(crayons.normal(u'Requirements file provided! Importing into Pipfileā€¦', bold=True), err=True) + click.echo( + crayons.normal( + u'Requirements file provided! Importing into Pipfileā€¦', bold=True + ), + err=True, + ) try: import_requirements(r=project.path_to(requirements), dev=dev) except (UnicodeDecodeError, pip9.exceptions.PipError) as e: # Don't print the temp file path if remote since it will be deleted. req_path = requirements_url if remote else project.path_to(requirements) - error = (u'Unexpected syntax in {0}. Are you sure this is a ' - 'requirements.txt style file?'.format(req_path)) + error = ( + u'Unexpected syntax in {0}. Are you sure this is a ' + 'requirements.txt style file?'.format(req_path) + ) traceback = e except AssertionError as e: - error = (u'Requirements file doesn\'t appear to exist. Please ensure the file exists in your ' - 'project directory or you provided the correct path.') + error = ( + u'Requirements file doesn\'t appear to exist. Please ensure the file exists in your ' + 'project directory or you provided the correct path.' + ) traceback = e finally: # If requirements file was provided by remote url delete the temporary file if remote: os.close(fd) # Close for windows to allow file cleanup. os.remove(project.path_to(temp_reqs)) - if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) requirements_directory.cleanup() sys.exit(1) - if code: - click.echo(crayons.normal(u'Discovering imports from local codebaseā€¦', bold=True)) + click.echo( + crayons.normal(u'Discovering imports from local codebaseā€¦', bold=True) + ) for req in import_from_code(code): click.echo(' Found {0}!'.format(crayons.green(req))) project.add_package_to_pipfile(req) - # Capture -e argument and assign it to following package_name. more_packages = list(more_packages) if package_name == '-e': package_name = ' '.join([package_name, more_packages.pop(0)]) - # Capture . argument and assign it to nothing if package_name == '.': package_name = False - # Allow more than one package to be provided. - package_names = [package_name, ] + more_packages - + package_names = [package_name] + more_packages # Install all dependencies, if none was provided. if package_name is False: # Update project settings with pre preference. if pre: project.update_settings({'allow_prereleases': pre}) - do_init( - dev=dev, allow_global=system, ignore_pipfile=ignore_pipfile, system=system, - skip_lock=skip_lock, verbose=verbose, concurrent=concurrent, deploy=deploy, - pre=pre, requirements_dir=requirements_directory + dev=dev, + allow_global=system, + ignore_pipfile=ignore_pipfile, + system=system, + skip_lock=skip_lock, + verbose=verbose, + concurrent=concurrent, + deploy=deploy, + pre=pre, + requirements_dir=requirements_directory, ) requirements_directory.cleanup() sys.exit(0) - # Support for --selective-upgrade. if selective_upgrade: - for i, package_name in enumerate(package_names[:]): section = project.packages if not dev else project.dev_packages package = convert_deps_from_pip(package_name) package__name = list(package.keys())[0] package__val = list(package.values())[0] - try: if not is_star(section[package__name]) and is_star(package__val): # Support for VCS dependencies. - package_names[i] = convert_deps_to_pip({package_name: section[package__name]}, r=False)[0] + package_names[i] = convert_deps_to_pip( + {package_name: section[package__name]}, r=False + )[ + 0 + ] except KeyError: pass - for package_name in package_names: - click.echo(crayons.normal(u'Installing {0}ā€¦'.format(crayons.green(package_name, bold=True)), bold=True)) - + click.echo( + crayons.normal( + u'Installing {0}ā€¦'.format(crayons.green(package_name, bold=True)), + bold=True, + ) + ) # pip install: with spinner(): - c = pip_install( package_name, ignore_hashes=True, @@ -1888,9 +1877,8 @@ def do_install( no_deps=False, verbose=verbose, pre=pre, - requirements_dir=requirements_directory.name + requirements_dir=requirements_directory.name, ) - # Warn if --editable wasn't passed. try: converted = convert_deps_from_pip(package_name) @@ -1898,34 +1886,32 @@ def do_install( click.echo('{0}: {1}'.format(crayons.red('WARNING'), e)) requirements_directory.cleanup() sys.exit(1) - key = [k for k in converted.keys()][0] - if is_vcs(key) or is_vcs(converted[key]) and not converted[key].get('editable'): + if is_vcs(key) or is_vcs(converted[key]) and not converted[key].get( + 'editable' + ): click.echo( '{0}: You installed a VCS dependency in nonā€“editable mode. ' 'This will work fine, but sub-dependencies will not be resolved by {1}.' '\n To enable this subā€“dependency functionality, specify that this dependency is editable.' ''.format( - crayons.red('Warning', bold=True), - crayons.red('$ pipenv lock') + crayons.red('Warning', bold=True), crayons.red('$ pipenv lock') ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. try: assert c.return_code == 0 except AssertionError: click.echo( '{0} An error occurred while installing {1}!'.format( - crayons.red('Error: ', bold=True), - crayons.green(package_name) - ), err=True) + crayons.red('Error: ', bold=True), crayons.green(package_name) + ), + err=True, + ) click.echo(crayons.blue(format_pip_error(c.err)), err=True) requirements_directory.cleanup() sys.exit(1) - click.echo( '{0} {1} {2} {3}{4}'.format( crayons.normal('Adding', bold=True), @@ -1935,42 +1921,49 @@ def do_install( crayons.normal('ā€¦', bold=True), ) ) - # Add the package to the Pipfile. try: project.add_package_to_pipfile(package_name, dev) except ValueError as e: - click.echo('{0} {1}'.format(crayons.red('ERROR (PACKAGE NOT INSTALLED):'), e)) - + click.echo( + '{0} {1}'.format(crayons.red('ERROR (PACKAGE NOT INSTALLED):'), e) + ) # Update project settings with pre preference. if pre: project.update_settings({'allow_prereleases': pre}) - if lock and not skip_lock: - do_init(dev=dev, allow_global=system, concurrent=concurrent, verbose=verbose, keep_outdated=keep_outdated, requirements_dir=requirements_directory) + do_init( + dev=dev, + allow_global=system, + concurrent=concurrent, + verbose=verbose, + keep_outdated=keep_outdated, + requirements_dir=requirements_directory, + ) requirements_directory.cleanup() def do_uninstall( - package_name=False, more_packages=False, three=None, python=False, - system=False, lock=False, all_dev=False, all=False, verbose=False, - keep_outdated=False + package_name=False, + more_packages=False, + three=None, + python=False, + system=False, + lock=False, + all_dev=False, + all=False, + verbose=False, + keep_outdated=False, ): - # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - # Ensure that virtualenv is available. ensure_project(three=three, python=python) - # Load the --pre settings from the Pipfile. pre = project.settings.get('allow_prereleases') - - package_names = (package_name,) + more_packages pipfile_remove = True - # Un-install all dependencies, if --all was provided. if all is True: click.echo( @@ -1978,52 +1971,39 @@ def do_uninstall( ) do_purge(allow_global=system, verbose=verbose) sys.exit(0) - # Uninstall [dev-packages], if --dev was provided. if all_dev: if 'dev-packages' not in project.parsed_pipfile: click.echo( - crayons.normal('No {0} to uninstall.'.format( - crayons.red('[dev-packages]')), bold=True + crayons.normal( + 'No {0} to uninstall.'.format(crayons.red('[dev-packages]')), + bold=True, ) ) sys.exit(0) - click.echo( - crayons.normal(u'Un-installing {0}ā€¦'.format( - crayons.red('[dev-packages]')), bold=True + crayons.normal( + u'Un-installing {0}ā€¦'.format(crayons.red('[dev-packages]')), bold=True ) ) package_names = project.parsed_pipfile['dev-packages'] package_names = package_names.keys() - if package_name is False and not all_dev: click.echo(crayons.red('No package provided!'), err=True) sys.exit(1) - for package_name in package_names: - - click.echo(u'Un-installing {0}ā€¦'.format( - crayons.green(package_name)) - ) - + click.echo(u'Un-installing {0}ā€¦'.format(crayons.green(package_name))) cmd = '"{0}" uninstall {1} -y'.format( - which_pip(allow_global=system), - package_name + which_pip(allow_global=system), package_name ) if verbose: click.echo('$ {0}'.format(cmd)) - c = delegator.run(cmd) - click.echo(crayons.blue(c.out)) - if pipfile_remove: norm_name = pep423_name(package_name) - in_dev_packages = (norm_name in project._pipfile.get('dev-packages', {})) in_packages = (norm_name in project._pipfile.get('packages', {})) - if not in_dev_packages and not in_packages: click.echo( 'No package {0} to remove from Pipfile.'.format( @@ -2033,15 +2013,11 @@ def do_uninstall( continue click.echo( - u'Removing {0} from Pipfileā€¦'.format( - crayons.green(package_name) - ) + u'Removing {0} from Pipfileā€¦'.format(crayons.green(package_name)) ) - # Remove package from both packages and dev-packages. project.remove_package_from_pipfile(package_name, dev=True) project.remove_package_from_pipfile(package_name, dev=False) - if lock: do_lock(system=system, pre=pre, keep_outdated=keep_outdated) @@ -2051,16 +2027,12 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False) - # Set an environment variable, so we know we're in the environment. os.environ['PIPENV_ACTIVE'] = '1' - compat = (not fancy) - # Support shell compatibility mode. if PIPENV_SHELL_FANCY: compat = False - # Compatibility mode: if compat: if PIPENV_SHELL: @@ -2069,23 +2041,24 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None): click.echo( crayons.red( 'Please ensure that the {0} environment variable ' - 'is set before activating shell.'.format(crayons.normal('SHELL', bold=True)) - ), err=True + 'is set before activating shell.'.format( + crayons.normal('SHELL', bold=True) + ) + ), + err=True, ) sys.exit(1) - click.echo( crayons.normal( 'Spawning environment shell ({0}). Use {1} to leave.'.format( - crayons.red(shell), - crayons.normal("'exit'", bold=True) - ), bold=True - ), err=True + crayons.red(shell), crayons.normal("'exit'", bold=True) + ), + bold=True, + ), + err=True, ) - cmd = "{0} -i'".format(shell) args = [] - # Standard (properly configured shell) mode: else: if PIPENV_VENV_IN_PROJECT: @@ -2093,28 +2066,20 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None): workon_name = '.venv' else: workon_name = project.virtualenv_name - cmd = sys.executable args = ['-m', 'pipenv.pew', 'workon', workon_name] - # Grab current terminal dimensions to replace the hardcoded default # dimensions of pexpect terminal_dimensions = get_terminal_size() - try: with temp_environ(): if PIPENV_VENV_IN_PROJECT: os.environ['WORKON_HOME'] = project.project_directory - c = pexpect.spawn( cmd, args, - dimensions=( - terminal_dimensions.lines, - terminal_dimensions.columns - ) + dimensions=(terminal_dimensions.lines, terminal_dimensions.columns), ) - # Windows! except AttributeError: # import subprocess @@ -2124,11 +2089,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None): os.environ['WORKON_HOME'] = project.project_directory pew.workon_cmd([workon_name]) sys.exit(0) - # Activate the virtualenv if in compatibility mode. if compat: c.sendline(activate_virtualenv()) - # Send additional arguments to the subshell. if shell_args: c.sendline(' '.join(shell_args)) @@ -2139,8 +2102,8 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None): def sigwinch_passthrough(sig, data): terminal_dimensions = get_terminal_size() c.setwinsize(terminal_dimensions.lines, terminal_dimensions.columns) - signal.signal(signal.SIGWINCH, sigwinch_passthrough) + signal.signal(signal.SIGWINCH, sigwinch_passthrough) # Interact with the new shell. c.interact(escape_character=None) c.close() @@ -2152,26 +2115,23 @@ def inline_activate_virtualenv(): activate_this = which('activate_this.py') with open(activate_this) as f: code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) + exec (code, dict(__file__=activate_this)) # Catch all errors, just in case. except Exception: click.echo( u'{0}: There was an unexpected error while activating your virtualenv. Continuing anywayā€¦' ''.format(crayons.red('Warning', bold=True)), - err=True + err=True, ) def do_run(command, args, three=None, python=False): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False) - load_dot_env() - # Script was foundā€¦ if command in project.scripts: command = ' '.join(project.scripts[command]) - # Separate out things that were passed in as a string. _c = list(command.split()) command = _c.pop(0) @@ -2179,14 +2139,15 @@ def do_run(command, args, three=None, python=False): args = list(args) for __c in reversed(_c): args.insert(0, __c) - # Activate virtualenv under the current interpreter's environment inline_activate_virtualenv() - # Windows! if os.name == 'nt': import subprocess - p = subprocess.Popen([command] + list(args), shell=True, universal_newlines=True) + + p = subprocess.Popen( + [command] + list(args), shell=True, universal_newlines=True + ) p.communicate() sys.exit(p.returncode) else: @@ -2198,63 +2159,58 @@ def do_run(command, args, three=None, python=False): crayons.red('Error', bold=True), crayons.red(command), crayons.normal('PATH', bold=True), - crayons.normal('[scripts]', bold=True) - ), err=True + crayons.normal('[scripts]', bold=True), + ), + err=True, ) sys.exit(1) - # Execute the command. os.execl(command_path, command_path, *args) pass def do_check(three=None, python=False, system=False, unused=False, args=None): - if not system: # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False, warn=False) - if not args: args = [] - if unused: deps_required = [k for k in project.packages.keys()] deps_needed = import_from_code(unused) - for dep in deps_needed: try: deps_required.remove(dep) except ValueError: pass - if deps_required: - click.echo(crayons.normal('The following dependencies appear unused, and may be safe for removal:')) + click.echo( + crayons.normal( + 'The following dependencies appear unused, and may be safe for removal:' + ) + ) for dep in deps_required: click.echo(' - {0}'.format(crayons.green(dep))) sys.exit(1) else: sys.exit(0) - - click.echo( - crayons.normal(u'Checking PEP 508 requirementsā€¦', bold=True) - ) - + click.echo(crayons.normal(u'Checking PEP 508 requirementsā€¦', bold=True)) if system: python = system_which('python') else: python = which('python') - # Run the PEP 508 checker in the virtualenv. - c = delegator.run('"{0}" {1}'.format(python, escape_grouped_arguments(pep508checker.__file__.rstrip('cdo')))) + c = delegator.run( + '"{0}" {1}'.format( + python, escape_grouped_arguments(pep508checker.__file__.rstrip('cdo')) + ) + ) results = simplejson.loads(c.out) - # Load the pipfile. p = pipfile.Pipfile.load(project.pipfile_location) - failed = False # Assert each specified requirement. for marker, specifier in p.data['_meta']['requires'].items(): - if marker in results: try: assert results[marker] == specifier @@ -2265,48 +2221,44 @@ def do_check(three=None, python=False, system=False, unused=False, args=None): ''.format( crayons.green(marker), crayons.blue(specifier), - crayons.red(results[marker]) - ), err=True + crayons.red(results[marker]), + ), + err=True, ) if failed: click.echo(crayons.red('Failed!'), err=True) sys.exit(1) else: click.echo(crayons.green('Passed!')) - - click.echo( - crayons.normal(u'Checking installed package safetyā€¦', bold=True) - ) - + click.echo(crayons.normal(u'Checking installed package safetyā€¦', bold=True)) path = pep508checker.__file__.rstrip('cdo') path = os.sep.join(__file__.split(os.sep)[:-1] + ['patched', 'safety.zip']) - if not system: python = which('python') else: python = system_which('python') - - c = delegator.run('"{0}" {1} check --json --key=1ab8d58f-5122e025-83674263-bc1e79e0'.format(python, escape_grouped_arguments(path))) + c = delegator.run( + '"{0}" {1} check --json --key=1ab8d58f-5122e025-83674263-bc1e79e0'.format( + python, escape_grouped_arguments(path) + ) + ) try: results = simplejson.loads(c.out) except ValueError: click.echo('An error occurred:', err=True) click.echo(c.err, err=True) sys.exit(1) - for (package, resolved, installed, description, vuln) in results: click.echo( '{0}: {1} {2} resolved ({3} installed)!'.format( crayons.normal(vuln, bold=True), crayons.green(package), crayons.red(resolved, bold=False), - crayons.red(installed, bold=True) + crayons.red(installed, bold=True), ) ) - click.echo('{0}'.format(description)) click.echo() - if not results: click.echo(crayons.green('All good!')) else: @@ -2322,26 +2274,25 @@ def do_graph(bare=False, json=False, reverse=False): crayons.red('Warning', bold=True), u'Unable to display currentlyā€“installed dependency graph information here. ' u'Please run within a Pipenv project.', - ), err=True + ), + err=True, ) sys.exit(1) - if reverse and json: click.echo( u'{0}: {1}'.format( crayons.red('Warning', bold=True), u'Using both --reverse and --json together is not supported. ' u'Please select one of the two options.', - ), err=True + ), + err=True, ) sys.exit(1) - flag = '' if json: flag = '--json' if reverse: flag = '--reverse' - if not project.virtualenv_exists: click.echo( u'{0}: No virtualenv has been created for this project yet! Consider ' @@ -2349,35 +2300,26 @@ def do_graph(bare=False, json=False, reverse=False): u'{2} for further instructions.'.format( crayons.red('Warning', bold=True), crayons.green('`pipenv install`'), - crayons.green('`pipenv install --help`') - ), err=True + crayons.green('`pipenv install --help`'), + ), + err=True, ) sys.exit(1) - - cmd = '"{0}" {1} {2}'.format( - python_path, - escape_grouped_arguments(pipdeptree.__file__.rstrip('cdo')), - flag + python_path, escape_grouped_arguments(pipdeptree.__file__.rstrip('cdo')), flag ) - # Run dep-tree. c = delegator.run(cmd) - if not bare: - if json: data = [] for d in simplejson.loads(c.out): - if d['package']['key'] not in BAD_PACKAGES: data.append(d) - click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) else: for line in c.out.split('\n'): - # Ignore bad packages as top level. if line.split('==')[0] in BAD_PACKAGES and not reverse: continue @@ -2385,13 +2327,11 @@ def do_graph(bare=False, json=False, reverse=False): # Bold top-level packages. if not line.startswith(' '): click.echo(crayons.normal(line, bold=True)) - # Echo the rest. else: click.echo(crayons.normal(line, bold=False)) else: click.echo(c.out) - # Return its return code. sys.exit(c.return_code) @@ -2409,79 +2349,67 @@ def do_sync( verbose=False, clear=False, unused=False, - sequential=False + sequential=False, ): requirements_dir = TemporaryDirectory(suffix='-requirements', prefix='pipenv-') # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False) - concurrent = (not sequential) - ensure_lockfile() - # Install everything. - do_init(dev=dev, verbose=verbose, concurrent=concurrent, requirements_dir=requirements_dir) - requirements_dir.cleanup() - - click.echo( - crayons.green('All dependencies are now up-to-date!') + do_init( + dev=dev, + verbose=verbose, + concurrent=concurrent, + requirements_dir=requirements_dir, ) + requirements_dir.cleanup() + click.echo(crayons.green('All dependencies are now up-to-date!')) -def do_clean( - ctx, - three=None, - python=None, - dry_run=False, - bare=False, - verbose=False -): - +def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, verbose=False): # Ensure that virtualenv is available. ensure_project(three=three, python=python, validate=False) - ensure_lockfile() - - installed_packages = delegator.run( - '{0} freeze'.format(which('pip')) - ).out.strip().split('\n') - + installed_packages = delegator.run('{0} freeze'.format(which('pip'))).out.strip( + ).split( + '\n' + ) installed_package_names = [] for installed in installed_packages: r = get_requirement(installed) - # Ignore editable installations. if not r.editable: installed_package_names.append(r.name.lower()) else: if verbose: click.echo('Ignoring {0}.'.format(repr(r.name)), err=True) - # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if bad_package in installed_package_names: if verbose: click.echo('Ignoring {0}.'.format(repr(bad_package)), err=True) del installed_package_names[installed_package_names.index(bad_package)] - # Intelligently detect if --dev should be used or not. develop = [k.lower() for k in project.lockfile_content['develop'].keys()] default = [k.lower() for k in project.lockfile_content['default'].keys()] - for used_package in set(develop + default): if used_package in installed_package_names: del installed_package_names[installed_package_names.index(used_package)] - failure = False for apparent_bad_package in installed_package_names: if dry_run: click.echo(apparent_bad_package) else: - click.echo(crayons.white('Uninstalling {0}ā€¦'.format(repr(apparent_bad_package)), bold=True)) - + click.echo( + crayons.white( + 'Uninstalling {0}ā€¦'.format(repr(apparent_bad_package)), bold=True + ) + ) # Uninstall the package. - c = delegator.run('{0} uninstall {1} -y'.format(which('pip'), apparent_bad_package)) + c = delegator.run( + '{0} uninstall {1} -y'.format(which('pip'), apparent_bad_package) + ) if c.return_code != 0: failure = True - sys.exit(int(failure)) diff --git a/pipenv/progress.py b/pipenv/progress.py index a2829aa77f..c38a23a394 100644 --- a/pipenv/progress.py +++ b/pipenv/progress.py @@ -1,186 +1,211 @@ -# -*- coding: utf-8 -*- - -""" -clint.textui.progress -~~~~~~~~~~~~~~~~~ - -This module provides the progressbar functionality. - -""" - -from __future__ import absolute_import - -import os -import sys -import time -import crayons - -STREAM = sys.stderr - -MILL_TEMPLATE = '%s %s %i/%i\r' - -DOTS_CHAR = '.' - -if os.name != 'nt': - BAR_FILLED_CHAR = str(crayons.green('ā–‰', bold=True)) - BAR_EMPTY_CHAR = str(crayons.black('ā–‰')) -else: - BAR_FILLED_CHAR = '=' - BAR_EMPTY_CHAR = '-' - -if (sys.version_info[0] >= 3) and (os.name != 'nt'): - BAR_TEMPLATE = u' %s%s%s %i/%i ā€” {0}\r'.format(crayons.black('%s')) -else: - if os.name == 'nt': - BAR_TEMPLATE = ' %s%s%s %i/%i - %s\r' - else: - BAR_TEMPLATE = ' %s%s%s %i/%i ā€” %s\r' - -MILL_CHARS = ['|', '/', '-', '\\'] - -# How long to wait before recalculating the ETA -ETA_INTERVAL = 1 -# How many intervals (excluding the current one) to calculate the simple moving -# average -ETA_SMA_WINDOW = 9 - - -class Bar(object): - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.done() - return False # we're not suppressing exceptions - - def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR, - filled_char=BAR_FILLED_CHAR, expected_size=None, every=1): - self.label = label - self.width = width - self.hide = hide - # Only show bar in terminals by default (better for piping, logging etc.) - if hide is None: - try: - self.hide = not STREAM.isatty() - except AttributeError: # output does not support isatty() - self.hide = True - self.empty_char = empty_char - self.filled_char = filled_char - self.expected_size = expected_size - self.every = every - self.start = time.time() - self.ittimes = [] - self.eta = 0 - self.etadelta = time.time() - self.etadisp = self.format_time(self.eta) - self.last_progress = 0 - if (self.expected_size): - self.show(0) - - def show(self, progress, count=None): - if count is not None: - self.expected_size = count - if self.expected_size is None: - raise Exception("expected_size not initialized") - self.last_progress = progress - if (time.time() - self.etadelta) > ETA_INTERVAL: - self.etadelta = time.time() - self.ittimes = \ - self.ittimes[-ETA_SMA_WINDOW:] + \ - [-(self.start - time.time()) / (progress+1)] - self.eta = \ - sum(self.ittimes) / float(len(self.ittimes)) * \ - (self.expected_size - progress) - self.etadisp = self.format_time(self.eta) - x = int(self.width * progress / self.expected_size) - if not self.hide: - if ((progress % self.every) == 0 or # True every "every" updates - (progress == self.expected_size)): # And when we're done - STREAM.write(BAR_TEMPLATE % ( - self.label, self.filled_char * x, - self.empty_char * (self.width - x), progress, - self.expected_size, self.etadisp)) - STREAM.flush() - - def done(self): - self.elapsed = time.time() - self.start - elapsed_disp = self.format_time(self.elapsed) - if not self.hide: - # Print completed bar with elapsed time - STREAM.write(BAR_TEMPLATE % ( - self.label, self.filled_char * self.width, - self.empty_char * 0, self.last_progress, - self.expected_size, elapsed_disp)) - STREAM.write('\n') - STREAM.flush() - - def format_time(self, seconds): - return time.strftime('%H:%M:%S', time.gmtime(seconds)) - - -def bar(it, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR, - filled_char=BAR_FILLED_CHAR, expected_size=None, every=1): - """Progress iterator. Wrap your iterables with it.""" - - count = len(it) if expected_size is None else expected_size - - with Bar(label=label, width=width, hide=hide, empty_char=BAR_EMPTY_CHAR, - filled_char=BAR_FILLED_CHAR, expected_size=count, every=every) \ - as bar: - for i, item in enumerate(it): - yield item - bar.show(i + 1) - - -def dots(it, label='', hide=None, every=1): - """Progress iterator. Prints a dot for each item being iterated""" - - count = 0 - - if not hide: - STREAM.write(label) - - for i, item in enumerate(it): - if not hide: - if i % every == 0: # True every "every" updates - STREAM.write(DOTS_CHAR) - sys.stderr.flush() - - count += 1 - - yield item - - STREAM.write('\n') - STREAM.flush() - - -def mill(it, label='', hide=None, expected_size=None, every=1): - """Progress iterator. Prints a mill while iterating over the items.""" - - def _mill_char(_i): - if _i >= count: - return ' ' - else: - return MILL_CHARS[(_i // every) % len(MILL_CHARS)] - - def _show(_i): - if not hide: - if ((_i % every) == 0 or # True every "every" updates - (_i == count)): # And when we're done - - STREAM.write(MILL_TEMPLATE % ( - label, _mill_char(_i), _i, count)) - STREAM.flush() - - count = len(it) if expected_size is None else expected_size - - if count: - _show(0) - - for i, item in enumerate(it): - yield item - _show(i + 1) - - if not hide: - STREAM.write('\n') - STREAM.flush() +# -*- coding: utf-8 -*- +""" +clint.textui.progress +~~~~~~~~~~~~~~~~~ + +This module provides the progressbar functionality. + +""" + +from __future__ import absolute_import + +import os +import sys +import time +import crayons + +STREAM = sys.stderr +MILL_TEMPLATE = '%s %s %i/%i\r' +DOTS_CHAR = '.' +if os.name != 'nt': + BAR_FILLED_CHAR = str(crayons.green('ā–‰', bold=True)) + BAR_EMPTY_CHAR = str(crayons.black('ā–‰')) +else: + BAR_FILLED_CHAR = '=' + BAR_EMPTY_CHAR = '-' +if (sys.version_info[0] >= 3) and (os.name != 'nt'): + BAR_TEMPLATE = u' %s%s%s %i/%i ā€” {0}\r'.format(crayons.black('%s')) +else: + if os.name == 'nt': + BAR_TEMPLATE = ' %s%s%s %i/%i - %s\r' + else: + BAR_TEMPLATE = ' %s%s%s %i/%i ā€” %s\r' +MILL_CHARS = ['|', '/', '-', '\\'] +# How long to wait before recalculating the ETA +ETA_INTERVAL = 1 +# How many intervals (excluding the current one) to calculate the simple moving +# average +ETA_SMA_WINDOW = 9 + + +class Bar(object): + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.done() + return False # we're not suppressing exceptions + + def __init__( + self, + label='', + width=32, + hide=None, + empty_char=BAR_EMPTY_CHAR, + filled_char=BAR_FILLED_CHAR, + expected_size=None, + every=1, + ): + self.label = label + self.width = width + self.hide = hide + # Only show bar in terminals by default (better for piping, logging etc.) + if hide is None: + try: + self.hide = not STREAM.isatty() + except AttributeError: # output does not support isatty() + self.hide = True + self.empty_char = empty_char + self.filled_char = filled_char + self.expected_size = expected_size + self.every = every + self.start = time.time() + self.ittimes = [] + self.eta = 0 + self.etadelta = time.time() + self.etadisp = self.format_time(self.eta) + self.last_progress = 0 + if (self.expected_size): + self.show(0) + + def show(self, progress, count=None): + if count is not None: + self.expected_size = count + if self.expected_size is None: + raise Exception("expected_size not initialized") + + self.last_progress = progress + if (time.time() - self.etadelta) > ETA_INTERVAL: + self.etadelta = time.time() + self.ittimes = self.ittimes[-ETA_SMA_WINDOW:] + [ + - (self.start - time.time()) / (progress + 1) + ] + self.eta = sum(self.ittimes) / float(len(self.ittimes)) * ( + self.expected_size - progress + ) + self.etadisp = self.format_time(self.eta) + x = int(self.width * progress / self.expected_size) + if not self.hide: + if ( + (progress % self.every) == 0 or # True every "every" updates + (progress == self.expected_size) + ): # And when we're done + STREAM.write( + BAR_TEMPLATE % + ( + self.label, + self.filled_char * x, + self.empty_char * (self.width - x), + progress, + self.expected_size, + self.etadisp, + ) + ) + STREAM.flush() + + def done(self): + self.elapsed = time.time() - self.start + elapsed_disp = self.format_time(self.elapsed) + if not self.hide: + # Print completed bar with elapsed time + STREAM.write( + BAR_TEMPLATE % + ( + self.label, + self.filled_char * self.width, + self.empty_char * 0, + self.last_progress, + self.expected_size, + elapsed_disp, + ) + ) + STREAM.write('\n') + STREAM.flush() + + def format_time(self, seconds): + return time.strftime('%H:%M:%S', time.gmtime(seconds)) + + +def bar( + it, + label='', + width=32, + hide=None, + empty_char=BAR_EMPTY_CHAR, + filled_char=BAR_FILLED_CHAR, + expected_size=None, + every=1, +): + """Progress iterator. Wrap your iterables with it.""" + count = len(it) if expected_size is None else expected_size + with Bar( + label=label, + width=width, + hide=hide, + empty_char=BAR_EMPTY_CHAR, + filled_char=BAR_FILLED_CHAR, + expected_size=count, + every=every, + ) as bar: + for i, item in enumerate(it): + yield item + + bar.show(i + 1) + + +def dots(it, label='', hide=None, every=1): + """Progress iterator. Prints a dot for each item being iterated""" + count = 0 + if not hide: + STREAM.write(label) + for i, item in enumerate(it): + if not hide: + if i % every == 0: # True every "every" updates + STREAM.write(DOTS_CHAR) + sys.stderr.flush() + count += 1 + yield item + + STREAM.write('\n') + STREAM.flush() + + +def mill(it, label='', hide=None, expected_size=None, every=1): + """Progress iterator. Prints a mill while iterating over the items.""" + + def _mill_char(_i): + if _i >= count: + return ' ' + + else: + return MILL_CHARS[(_i // every) % len(MILL_CHARS)] + + def _show(_i): + if not hide: + if ( + (_i % every) == 0 or (_i == count) # True every "every" updates + ): # And when we're done + STREAM.write(MILL_TEMPLATE % (label, _mill_char(_i), _i, count)) + STREAM.flush() + + count = len(it) if expected_size is None else expected_size + if count: + _show(0) + for i, item in enumerate(it): + yield item + + _show(i + 1) + if not hide: + STREAM.write('\n') + STREAM.flush() diff --git a/pipenv/project.py b/pipenv/project.py index 88d83118be..61261ea9c3 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,10 +14,20 @@ from pip9 import ConfigOptionParser from .utils import ( - mkdir_p, convert_deps_from_pip, pep423_name, recase_file, - find_requirements, is_file, is_vcs, python_version, cleanup_toml, - is_installable_file, is_valid_url, normalize_drive, python_version, - escape_grouped_arguments + mkdir_p, + convert_deps_from_pip, + pep423_name, + recase_file, + find_requirements, + is_file, + is_vcs, + python_version, + cleanup_toml, + is_installable_file, + is_valid_url, + normalize_drive, + python_version, + escape_grouped_arguments, ) from .environments import ( PIPENV_MAX_DEPTH, @@ -26,12 +36,13 @@ PIPENV_VIRTUALENV, PIPENV_NO_INHERIT, PIPENV_TEST_INDEX, - PIPENV_PYTHON + PIPENV_PYTHON, ) if PIPENV_PIPFILE: if not os.path.isfile(PIPENV_PIPFILE): raise RuntimeError('Given PIPENV_PIPFILE is not found!') + else: PIPENV_PIPFILE = normalize_drive(os.path.abspath(PIPENV_PIPFILE)) @@ -50,7 +61,6 @@ def __init__(self, which=None, python_version=None, chdir=True): self._original_dir = os.path.abspath(os.curdir) self.which = which self.python_version = python_version - # Hack to skip this during pipenv run, or -r. if ('run' not in sys.argv) and chdir: try: @@ -75,13 +85,22 @@ def _build_package_list(self, package_section): # When a vcs url is gven without editable it only appears as a key # Eliminate any vcs, path, or url entries which are not editable # Since pip-tools can't do deep resolution on them, even setuptools-installable ones - if (is_vcs(v) or is_vcs(k) or (is_installable_file(k) or is_installable_file(v)) or - any((prefix in v and - (os.path.isfile(v[prefix]) or is_valid_url(v[prefix]))) - for prefix in ['path', 'file'])): + if ( + is_vcs(v) or + is_vcs(k) or + (is_installable_file(k) or is_installable_file(v)) or + any( + ( + prefix in v and + (os.path.isfile(v[prefix]) or is_valid_url(v[prefix])) + ) + for prefix in ['path', 'file'] + ) + ): # If they are editable, do resolve them if 'editable' not in v: continue + else: ps.update({k: v}) else: @@ -90,13 +109,15 @@ def _build_package_list(self, package_section): # Since these entries have no attributes we know they are not editable # So we can safely exclude things that need to be editable in order to be resolved # First exclude anything that is a vcs entry either in the key or value - if not (any(is_vcs(i) for i in [k, v]) or - # Then exclude any installable files that are not directories - # Because pip-tools can resolve setup.py for example - any(is_installable_file(i) for i in [k, v]) or - # Then exclude any URLs because they need to be editable also - # Things that are excluded can only be 'shallow resolved' - any(is_valid_url(i) for i in [k, v])): + if not ( + any(is_vcs(i) for i in [k, v]) or + # Then exclude any installable files that are not directories + # Because pip-tools can resolve setup.py for example + any(is_installable_file(i) for i in [k, v]) or + # Then exclude any URLs because they need to be editable also + # Things that are excluded can only be 'shallow resolved' + any(is_valid_url(i) for i in [k, v]) + ): ps.update({k: v}) return ps @@ -113,7 +134,9 @@ def pipfile_exists(self): @property def required_python_version(self): if self.pipfile_exists: - required = self.parsed_pipfile.get('requires', {}).get('python_full_version') + required = self.parsed_pipfile.get('requires', {}).get( + 'python_full_version' + ) if not required: required = self.parsed_pipfile.get('requires', {}).get('python_version') if required != "*": @@ -123,6 +146,7 @@ def required_python_version(self): def project_directory(self): if self.pipfile_location is not None: return os.path.abspath(os.path.join(self.pipfile_location, os.pardir)) + else: return None @@ -157,22 +181,19 @@ def virtualenv_name(self): # http://www.tldp.org/LDP/abs/html/special-chars.html#FIELDREF # https://github.com/torvalds/linux/blob/2bfe01ef/include/uapi/linux/binfmts.h#L18 sanitized = re.sub(r'[ $`!*@"\\\r\n\t]', '_', self.name)[0:42] - # Hash the full path of the pipfile hash = hashlib.sha256(self.pipfile_location.encode()).digest()[:6] encoded_hash = base64.urlsafe_b64encode(hash).decode() - # If the pipfile was located at '/home/user/MY_PROJECT/Pipfile', # the name of its virtualenv will be 'my-project-wyUfYPqE' - if PIPENV_PYTHON: return sanitized + '-' + encoded_hash + '-' + PIPENV_PYTHON + else: return sanitized + '-' + encoded_hash @property def virtualenv_location(self): - # if VIRTUAL_ENV is set, use that. if PIPENV_VIRTUALENV: return PIPENV_VIRTUALENV @@ -183,12 +204,15 @@ def virtualenv_location(self): # The user wants the virtualenv in the project. if not PIPENV_VENV_IN_PROJECT: - c = delegator.run('{0} -m pipenv.pew dir "{1}"'.format(escape_grouped_arguments(sys.executable), self.virtualenv_name)) + c = delegator.run( + '{0} -m pipenv.pew dir "{1}"'.format( + escape_grouped_arguments(sys.executable), self.virtualenv_name + ) + ) loc = c.out.strip() # Default mode. else: loc = os.sep.join(self.pipfile_location.split(os.sep)[:-1] + ['.venv']) - self._virtualenv_location = loc return loc @@ -203,10 +227,8 @@ def download_location(self): if self._download_location is None: loc = os.sep.join([self.virtualenv_location, 'downloads']) self._download_location = loc - # Create the directory, if it doesn't exist. mkdir_p(self._download_location) - return self._download_location @property @@ -214,10 +236,8 @@ def proper_names_location(self): if self._proper_names_location is None: loc = os.sep.join([self.virtualenv_location, 'pipenv-proper-names.txt']) self._proper_names_location = loc - # Create the database, if it doesn't exist. open(self._proper_names_location, 'a').close() - return self._proper_names_location @property @@ -241,7 +261,6 @@ def pipfile_location(self): except RuntimeError: loc = None self._pipfile_location = normalize_drive(loc) - return self._pipfile_location @property @@ -252,7 +271,6 @@ def requirements_location(self): except RuntimeError: loc = None self._requirements_location = loc - return self._requirements_location @property @@ -260,25 +278,21 @@ def parsed_pipfile(self): # Open the pipfile, read it into memory. with open(self.pipfile_location) as f: contents = f.read() - # If any outline tables are present... if ('[packages.' in contents) or ('[dev-packages.' in contents): - data = toml.loads(contents) - # Convert all outline tables to inline tables. for section in ('packages', 'dev-packages'): for package in data.get(section, {}): - # Convert things to inline tables ā€” fancy :) if hasattr(data[section][package], 'keys'): _data = data[section][package] data[section][package] = toml._get_empty_inline_table(dict) data[section][package].update(_data) - # We lose comments here, but it's for the best.) try: return contoml.loads(toml.dumps(data, preserve=True)) + except RuntimeError: return toml.loads(toml.dumps(data, preserve=True)) @@ -286,6 +300,7 @@ def parsed_pipfile(self): # Fallback to toml parser, for large files. try: return contoml.loads(contents) + except Exception: return toml.loads(contents) @@ -295,12 +310,10 @@ def _pipfile(self): pfile = self.parsed_pipfile for section in ('packages', 'dev-packages'): p_section = pfile.get(section, {}) - for key in list(p_section.keys()): # Normalize key name to PEP 423. norm_key = pep423_name(key) p_section[norm_key] = p_section.pop(key) - return pfile @property @@ -313,23 +326,18 @@ def scripts(self): scripts = self.parsed_pipfile.get('scripts', {}) for (k, v) in scripts.items(): scripts[k] = shlex.split(v, posix=True) - return scripts - def update_settings(self, d): settings = self.settings - changed = False for new in d: if new not in settings: settings[new] = d[new] changed = True - if changed: p = self.parsed_pipfile p['pipenv'] = settings - # Write the changes to disk. self.write_toml(p) @@ -338,14 +346,11 @@ def _lockfile(self): """Pipfile.lock divided by PyPI and external dependencies.""" pfile = pipfile.load(self.pipfile_location) lockfile = json.loads(pfile.lock()) - for section in ('default', 'develop'): lock_section = lockfile.get(section, {}) - for key in list(lock_section.keys()): norm_key = pep423_name(key) lockfile[section][norm_key] = lock_section.pop(key) - return lockfile @property @@ -417,56 +422,55 @@ def create_pipfile(self, python=None): config_parser = ConfigOptionParser(name=self.name) install = dict(config_parser.get_config_section('install')) indexes = install.get('extra-index-url', '').lstrip('\n').split('\n') - if PIPENV_TEST_INDEX: - sources = [{u'url': PIPENV_TEST_INDEX, u'verify_ssl': True, u'name': u'custom'}] + sources = [ + {u'url': PIPENV_TEST_INDEX, u'verify_ssl': True, u'name': u'custom'} + ] else: # Default source. - pypi_source = {u'url': u'https://pypi.python.org/simple', u'verify_ssl': True, u'name': 'pypi'} + pypi_source = { + u'url': u'https://pypi.python.org/simple', + u'verify_ssl': True, + u'name': 'pypi', + } sources = [pypi_source] - for i, index in enumerate(indexes): if not index: continue + source_name = 'pip_index_{}'.format(i) verify_ssl = index.startswith('https') - - sources.append({u'url': index, u'verify_ssl': verify_ssl, u'name': source_name}) - + sources.append( + {u'url': index, u'verify_ssl': verify_ssl, u'name': source_name} + ) data = { u'source': sources, - # Default packages. u'packages': {}, u'dev-packages': {}, - } - # Default requires. required_python = python or self.which('python', self.virtualenv_location) - data[u'requires'] = {'python_version': python_version(required_python)[:len('2.7')]} - + data[u'requires'] = { + 'python_version': python_version(required_python)[: len('2.7')] + } self.write_toml(data, 'Pipfile') def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: path = self.pipfile_location - try: formatted_data = contoml.dumps(data).rstrip() except Exception: for section in ('packages', 'dev-packages'): for package in data[section]: - # Convert things to inline tables ā€” fancy :) if hasattr(data[section][package], 'keys'): _data = data[section][package] data[section][package] = toml._get_empty_inline_table(dict) data[section][package].update(_data) - formatted_data = toml.dumps(data).rstrip() - formatted_data = cleanup_toml(formatted_data) with open(path, 'w') as f: f.write(formatted_data) @@ -478,16 +482,25 @@ def sources(self): sources_ = meta_.get('sources') if sources_: return sources_ + if 'source' in self.parsed_pipfile: return self.parsed_pipfile['source'] + else: - return [{u'url': u'https://pypi.python.org/simple', u'verify_ssl': True, 'name': 'pypi'}] + return [ + { + u'url': u'https://pypi.python.org/simple', + u'verify_ssl': True, + 'name': 'pypi', + } + ] def get_source(self, name=None, url=None): for source in self.sources: if name: if source.get('name') == name: return source + elif url: if source.get('url') in url: return source @@ -496,66 +509,49 @@ def destroy_lockfile(self): """Deletes the lockfile.""" try: return os.remove(self.lockfile_location) + except OSError: pass def remove_package_from_pipfile(self, package_name, dev=False): - # Read and append Pipfile. p = self._pipfile - package_name = pep423_name(package_name) - key = 'dev-packages' if dev else 'packages' - if key in p and package_name in p[key]: del p[key][package_name] - # Write Pipfile. self.write_toml(recase_file(p)) def add_package_to_pipfile(self, package_name, dev=False): - # Read and append Pipfile. p = self._pipfile - # Don't re-capitalize file URLs or VCSs. converted = convert_deps_from_pip(package_name) converted = converted[[k for k in converted.keys()][0]] - if not (is_file(package_name) or is_vcs(converted) or 'path' in converted): package_name = pep423_name(package_name) - key = 'dev-packages' if dev else 'packages' - # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - package = convert_deps_from_pip(package_name) package_name = [k for k in package.keys()][0] - # Add the package to the group. p[key][package_name] = package[package_name] - # Write Pipfile. self.write_toml(p) def add_index_to_pipfile(self, index): """Adds a given index to the Pipfile.""" - # Read and append Pipfile. p = self._pipfile - source = {'url': index, 'verify_ssl': True} - # Add the package to the group. if 'source' not in p: p['source'] = [source] - else: p['source'].append(source) - # Write Pipfile. self.write_toml(p) diff --git a/pipenv/utils.py b/pipenv/utils.py index 7d8ea8f62d..fc41dffaa0 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -15,6 +15,7 @@ import six import stat import warnings + try: from weakref import finalize except ImportError: @@ -22,11 +23,9 @@ from backports.weakref import finalize except ImportError: pass - from time import time logging.basicConfig(level=logging.ERROR) - try: from urllib.parse import urlparse except ImportError: @@ -38,8 +37,6 @@ from pathlib2 import Path except ImportError: pass - - from distutils.spawn import find_executable from contextlib import contextmanager from pipenv.patched.piptools.resolver import Resolver @@ -56,21 +53,22 @@ from .environments import SESSION_IS_INTERACTIVE, PIPENV_MAX_ROUNDS, PIPENV_CACHE_DIR if six.PY2: + class ResourceWarning(Warning): pass -specifiers = [k for k in lookup.keys()] +specifiers = [k for k in lookup.keys()] # List of version control systems we support. VCS_LIST = ('git', 'svn', 'hg', 'bzr') SCHEME_LIST = ('http://', 'https://', 'ftp://', 'file://') - requests = requests.Session() def get_requirement(dep): from pip9.req.req_install import _strip_extras import requirements + """Pre-clean requirement strings passed to the requirements parser. Ensures that we can accept both local and relative paths, file and VCS URIs, @@ -148,22 +146,21 @@ def get_requirement(dep): req.markers = markers if extras: # Bizarrely this is also what pip does... - req.extras = [r for r in requirements.parse('fakepkg{0}'.format(extras))][0].extras + req.extras = [r for r in requirements.parse('fakepkg{0}'.format(extras))][ + 0 + ].extras return req def cleanup_toml(tml): toml = tml.split('\n') new_toml = [] - # Remove all empty lines from TOML. for line in toml: if line.strip(): new_toml.append(line) - toml = '\n'.join(new_toml) new_toml = [] - # Add newlines between TOML sections. for i, line in enumerate(toml.split('\n')): after = False @@ -173,12 +170,10 @@ def cleanup_toml(tml): # Insert a newline before the heading. new_toml.append('\n') after = True - new_toml.append(line) # Insert a newline after the heading. if after: new_toml.append('') - # adding new line at the end of the TOML file new_toml.append('') toml = '\n'.join(new_toml) @@ -193,6 +188,7 @@ def python_version(path_to_python): c = delegator.run([path_to_python, '--version'], block=False) except Exception: return None + output = c.out.strip() or c.err.strip() @parse.with_pattern(r'.*') @@ -216,10 +212,10 @@ def escape_grouped_arguments(s): """ if s is None: return None + # Additional escaping for windows paths if os.name == 'nt': s = "{}".format(s.replace("\\", "\\\\")) - return '"' + s.replace("'", "'\\''") + '"' @@ -230,6 +226,7 @@ def clean_pkg_version(version): class HackedPythonVersion(object): """A Beautiful hack, which allows us to tell pip which version of Python we're using.""" + def __init__(self, python_version, python_path): self.python_version = python_version self.python_path = python_path @@ -246,28 +243,29 @@ def __exit__(self, *args): def prepare_pip_source_args(sources, pip_args=None): if pip_args is None: pip_args = [] - if sources: # Add the source to pip9. pip_args.extend(['-i', sources[0]['url']]) - # Trust the host if it's not verified. if not sources[0].get('verify_ssl', True): - pip_args.extend(['--trusted-host', urlparse(sources[0]['url']).netloc.split(':')[0]]) - + pip_args.extend( + ['--trusted-host', urlparse(sources[0]['url']).netloc.split(':')[0]] + ) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: pip_args.extend(['--extra-index-url', source['url']]) - # Trust the host if it's not verified. if not source.get('verify_ssl', True): - pip_args.extend(['--trusted-host', urlparse(source['url']).netloc.split(':')[0]]) - + pip_args.extend( + ['--trusted-host', urlparse(source['url']).netloc.split(':')[0]] + ) return pip_args -def actually_resolve_reps(deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre): +def actually_resolve_reps( + deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre +): from pip9 import basecommand, req from pip9._vendor import requests as pip_requests @@ -276,52 +274,51 @@ class PipCommand(basecommand.Command): name = 'PipCommand' constraints = [] - req_dir = tempfile.mkdtemp(prefix='pipenv-', suffix='-requirements') for dep in deps: if dep: if dep.startswith('-e '): constraint = req.InstallRequirement.from_editable(dep[len('-e '):]) else: - fd, t = tempfile.mkstemp(prefix='pipenv-', suffix='-requirement.txt', dir=req_dir) + fd, t = tempfile.mkstemp( + prefix='pipenv-', suffix='-requirement.txt', dir=req_dir + ) with os.fdopen(fd, 'w') as f: f.write(dep) - - constraint = [c for c in req.parse_requirements(t, session=pip_requests)][0] - - # extra_constraints = [] + constraint = [ + c for c in req.parse_requirements(t, session=pip_requests) + ][ + 0 + ] + # extra_constraints = [] if ' -i ' in dep: - index_lookup[constraint.name] = project.get_source(url=dep.split(' -i ')[1]).get('name') - + index_lookup[constraint.name] = project.get_source( + url=dep.split(' -i ')[1] + ).get( + 'name' + ) if constraint.markers: - markers_lookup[constraint.name] = str(constraint.markers).replace('"', "'") - + markers_lookup[constraint.name] = str(constraint.markers).replace( + '"', "'" + ) constraints.append(constraint) - rmtree(req_dir) - pip_command = get_pip_command() - pip_args = [] - if sources: pip_args = prepare_pip_source_args(sources, pip_args) - if verbose: print('Using pip: {0}'.format(' '.join(pip_args))) - pip_options, _ = pip_command.parse_args(pip_args) - session = pip_command._build_session(pip_options) pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) - if verbose: logging.log.verbose = True piptools_logging.log.verbose = True - resolved_tree = set() - - resolver = Resolver(constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre) + resolver = Resolver( + constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre + ) # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages try: resolved_tree.update(resolver.resolve(max_rounds=PIPENV_MAX_ROUNDS)) @@ -332,32 +329,36 @@ class PipCommand(basecommand.Command): ''.format( crayons.red('Warning', bold=True), crayons.red('$ pipenv install --skip-lock'), - crayons.red('$ pipenv graph') + crayons.red('$ pipenv graph'), ), - err=True) - + err=True, + ) click.echo(crayons.blue(str(e)), err=True) - if 'no version found at all' in str(e): - click.echo(crayons.blue('Please check your version specifier and version number. See PEP440 for more information.')) - + click.echo( + crayons.blue( + 'Please check your version specifier and version number. See PEP440 for more information.' + ) + ) raise RuntimeError return resolved_tree, resolver def venv_resolve_deps(deps, which, project, pre=False, verbose=False, clear=False): - from . import resolver + from .import resolver import json resolver = escape_grouped_arguments(resolver.__file__.rstrip('co')) - cmd = '{0} {1} {2} {3}'.format(escape_grouped_arguments(which('python')), resolver, '--pre' if pre else '', '--verbose' if verbose else '') + cmd = '{0} {1} {2} {3}'.format( + escape_grouped_arguments(which('python')), + resolver, + '--pre' if pre else '', + '--verbose' if verbose else '', + ) os.environ['PIPENV_PACKAGES'] = '\n'.join(deps) - c = delegator.run(cmd, block=True) - del os.environ['PIPENV_PACKAGES'] - try: assert c.return_code == 0 except AssertionError: @@ -367,125 +368,138 @@ def venv_resolve_deps(deps, which, project, pre=False, verbose=False, clear=Fals else: click.echo(c.err[int(len(c.err) / 2) - 1:], err=True) sys.exit(c.return_code) - if verbose: click.echo(c.out.split('RESULTS:')[0], err=True) - try: return json.loads(c.out.split('RESULTS:')[1].strip()) + except IndexError: raise RuntimeError('There was a problem with locking.') -def resolve_deps(deps, which, project, sources=None, verbose=False, python=False, clear=False, pre=False, allow_global=False): +def resolve_deps( + deps, + which, + project, + sources=None, + verbose=False, + python=False, + clear=False, + pre=False, + allow_global=False, +): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip9. """ - index_lookup = {} markers_lookup = {} - python_path = which('python', allow_global=allow_global) backup_python_path = sys.executable - results = [] - # First (proper) attempt: with HackedPythonVersion(python_version=python, python_path=python_path): - try: - resolved_tree, resolver = actually_resolve_reps(deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre) + resolved_tree, resolver = actually_resolve_reps( + deps, + index_lookup, + markers_lookup, + project, + sources, + verbose, + clear, + pre, + ) except RuntimeError: # Don't exit here, like usual. resolved_tree = None - # Second (last-resort) attempt: if resolved_tree is None: - with HackedPythonVersion(python_version='.'.join([str(s) for s in sys.version_info[:3]]), python_path=backup_python_path): - + with HackedPythonVersion( + python_version='.'.join([str(s) for s in sys.version_info[:3]]), + python_path=backup_python_path, + ): try: # Attempt to resolve again, with different Python version information, # particularly for particularly particular packages. - resolved_tree, resolver = actually_resolve_reps(deps, index_lookup, markers_lookup, project, sources, verbose, clear, pre) + resolved_tree, resolver = actually_resolve_reps( + deps, + index_lookup, + markers_lookup, + project, + sources, + verbose, + clear, + pre, + ) except RuntimeError: sys.exit(1) - - for result in resolved_tree: if not result.editable: name = pep423_name(result.name) version = clean_pkg_version(result.specifier) index = index_lookup.get(result.name) - if not markers_lookup.get(result.name): - markers = str(result.markers) if result.markers and 'extra' not in str(result.markers) else None + markers = str(result.markers) if result.markers and 'extra' not in str( + result.markers + ) else None else: markers = markers_lookup.get(result.name) - collected_hashes = [] if 'python.org' in '|'.join([source['url'] for source in sources]): try: # Grab the hashes from the new warehouse API. - r = requests.get('https://pypi.org/pypi/{0}/json'.format(name), timeout=10) + r = requests.get( + 'https://pypi.org/pypi/{0}/json'.format(name), timeout=10 + ) api_releases = r.json()['releases'] - cleaned_releases = {} for api_version, api_info in api_releases.items(): cleaned_releases[clean_pkg_version(api_version)] = api_info - for release in cleaned_releases[version]: collected_hashes.append(release['digests']['sha256']) collected_hashes = ['sha256:' + s for s in collected_hashes] - except (ValueError, KeyError, ConnectionError) as e: if verbose: - click.echo('{0}: Error generating hash for {1}'.format(crayons.red('Warning', bold=True), name)) - + click.echo( + '{0}: Error generating hash for {1}'.format( + crayons.red('Warning', bold=True), name + ) + ) # Collect un-collectable hashes (should work with devpi). try: - collected_hashes = collected_hashes + list(list(resolver.resolve_hashes([result]).items())[0][1]) + collected_hashes = collected_hashes + list( + list(resolver.resolve_hashes([result]).items())[0][1] + ) except (ValueError, KeyError, ConnectionError, IndexError): if verbose: print('Error generating hash for {}'.format(name)) - collected_hashes = list(set(collected_hashes)) - d = {'name': name, 'version': version, 'hashes': collected_hashes} - if index: d.update({'index': index}) - if markers: d.update({'markers': markers.replace('"', "'")}) - results.append(d) - return results def multi_split(s, split): """Splits on multiple given separators.""" - for r in split: s = s.replace(r, '|') - return [i for i in s.split('|') if len(i) > 0] def convert_deps_from_pip(dep): """"Converts a pip-formatted dependency to a Pipfile-formatted one.""" - dependency = {} - req = get_requirement(dep) extras = {'extras': req.extras} - # File installs. if (req.uri or req.path or is_installable_file(req.name)) and not req.vcs: # Assign a package name to the file, last 7 of it's sha256 hex digest. if not req.uri and not req.path: req.path = os.path.abspath(req.name) - hashable_path = req.uri if req.uri else req.path req.name = hashlib.sha256(hashable_path.encode('utf-8')).hexdigest() req.name = req.name[len(req.name) - 7:] @@ -494,68 +508,56 @@ def convert_deps_from_pip(dep): dependency[req.name] = {'file': hashable_path} else: dependency[req.name] = {'path': hashable_path} - if req.extras: dependency[req.name].update(extras) - # Add --editable if applicable if req.editable: dependency[req.name].update({'editable': True}) - # VCS Installs. Extra check for unparsed git over SSH elif req.vcs or is_vcs(req.path): if req.name is None: - raise ValueError('pipenv requires an #egg fragment for version controlled ' - 'dependencies. Please install remote dependency ' - 'in the form {0}#egg=.'.format(req.uri)) + raise ValueError( + 'pipenv requires an #egg fragment for version controlled ' + 'dependencies. Please install remote dependency ' + 'in the form {0}#egg=.'.format(req.uri) + ) # Set up this requirement as a proper VCS requirement if it was not if not req.vcs and req.path.startswith(VCS_LIST): req.vcs = [vcs for vcs in VCS_LIST if req.path.startswith(vcs)][0] req.uri = '{0}'.format(req.path) req.path = None - # Crop off the git+, etc part. if req.uri.startswith('{0}+'.format(req.vcs)): req.uri = req.uri[len(req.vcs) + 1:] dependency.setdefault(req.name, {}).update({req.vcs: req.uri}) - # Add --editable, if it's there. if req.editable: dependency[req.name].update({'editable': True}) - # Add subdirectory, if it's there if req.subdirectory: dependency[req.name].update({'subdirectory': req.subdirectory}) - # Add the specifier, if it was provided. if req.revision: dependency[req.name].update({'ref': req.revision}) - # Extras: e.g. #egg=requests[security] if req.extras: dependency[req.name].update({'extras': req.extras}) - elif req.extras or req.specs: - specs = None # Comparison operators: e.g. Django>1.10 if req.specs: r = multi_split(dep, '!=<>~') specs = dep[len(r[0]):] dependency[req.name] = specs - # Extras: e.g. requests[socks] if req.extras: dependency[req.name] = extras - if specs: dependency[req.name].update({'version': specs}) - # Bare dependencies: e.g. requests else: dependency[dep] = '*' - # Cleanup when there's multiple values, e.g. -e. if len(dependency) > 1: for key in dependency.copy(): @@ -563,45 +565,41 @@ def convert_deps_from_pip(dep): del dependency[key] return dependency + def is_star(val): return isinstance(val, six.string_types) and val == '*' + def is_pinned(val): return isinstance(val, six.string_types) and val.startswith('==') + def convert_deps_to_pip(deps, project=None, r=True, include_index=False): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - dependencies = [] - for dep in deps.keys(): - # Default (e.g. '>1.10'). extra = deps[dep] if isinstance(deps[dep], six.string_types) else '' version = '' index = '' - # Get rid of '*'. if is_star(deps[dep]) or str(extra) == '{}': extra = '' - hash = '' # Support for single hash (spec 1). if 'hash' in deps[dep]: hash = ' --hash={0}'.format(deps[dep]['hash']) - # Support for multiple hashes (spec 2). if 'hashes' in deps[dep]: - hash = '{0} '.format(''.join([' --hash={0} '.format(h) for h in deps[dep]['hashes']])) - + hash = '{0} '.format( + ''.join([' --hash={0} '.format(h) for h in deps[dep]['hashes']]) + ) # Support for extras (e.g. requests[socks]) if 'extras' in deps[dep]: extra = '[{0}]'.format(','.join(deps[dep]['extras'])) - if 'version' in deps[dep]: if not is_star(deps[dep]['version']): version = deps[dep]['version'] - # For lockfile format. if 'markers' in deps[dep]: specs = '; {0}'.format(deps[dep]['markers']) @@ -616,7 +614,6 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False): specs = '; {0}'.format(' and '.join(specs)) else: specs = '' - if include_index and not is_file(deps[dep]) and not is_vcs(deps[dep]): pip_src_args = [] if 'index' in deps[dep]: @@ -625,52 +622,42 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False): pip_src_args = project.sources pip_args = prepare_pip_source_args(pip_src_args) index = ' '.join(pip_args) - # Support for version control maybe_vcs = [vcs for vcs in VCS_LIST if vcs in deps[dep]] vcs = maybe_vcs[0] if maybe_vcs else None - # Support for files. if 'file' in deps[dep]: extra = '{1}{0}'.format(extra, deps[dep]['file']).strip() - # Flag the file as editable if it is a local relative path if 'editable' in deps[dep]: dep = '-e ' else: dep = '' - # Support for paths. elif 'path' in deps[dep]: extra = '{1}{0}'.format(extra, deps[dep]['path']).strip() - # Flag the file as editable if it is a local relative path if 'editable' in deps[dep]: dep = '-e ' else: dep = '' - if vcs: extra = '{0}+{1}'.format(vcs, deps[dep][vcs]) - # Support for @refs. if 'ref' in deps[dep]: extra += '@{0}'.format(deps[dep]['ref']) - extra += '#egg={0}'.format(dep) - # Support for subdirectory if 'subdirectory' in deps[dep]: extra += '&subdirectory={0}'.format(deps[dep]['subdirectory']) - # Support for editable. if 'editable' in deps[dep]: # Support for --egg. dep = '-e ' else: dep = '' - - s = '{0}{1}{2}{3}{4} {5}'.format(dep, extra, version, specs, hash, index).strip() + s = '{0}{1}{2}{3}{4} {5}'.format(dep, extra, version, specs, hash, index).strip( + ) dependencies.append(s) if not r: return dependencies @@ -689,11 +676,15 @@ def mkdir_p(newdir): - parent directory(ies) does not exist, make them as well From: http://code.activestate.com/recipes/82465-a-friendly-mkdir/ """ - if os.path.isdir(newdir): pass elif os.path.isfile(newdir): - raise OSError("a file with the same name as the desired dir, '{0}', already exists.".format(newdir)) + raise OSError( + "a file with the same name as the desired dir, '{0}', already exists.".format( + newdir + ) + ) + else: head, tail = os.path.split(newdir) if head and not os.path.isdir(head): @@ -706,12 +697,12 @@ def is_required_version(version, specified_version): """Check to see if there's a hard requirement for version number provided in the Pipfile. """ - # Certain packages may be defined with multiple values. if isinstance(specified_version, dict): specified_version = specified_version.get('version', '') if specified_version.startswith('=='): return version.strip() == specified_version.split('==')[1].strip() + return True @@ -733,12 +724,16 @@ def clean_git_uri(uri): def is_vcs(pipfile_entry): import requirements - """Determine if dictionary entry from Pipfile is for a vcs dependency.""" + """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if hasattr(pipfile_entry, 'keys'): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) + elif isinstance(pipfile_entry, six.string_types): - return bool(requirements.requirement.VCS_REGEX.match(clean_git_uri(pipfile_entry))) + return bool( + requirements.requirement.VCS_REGEX.match(clean_git_uri(pipfile_entry)) + ) + return False @@ -746,10 +741,14 @@ def is_installable_file(path): """Determine if a path can potentially be installed""" from pip9.utils import is_installable_dir from pip9.utils.packaging import specifiers - if hasattr(path, 'keys') and any(key for key in path.keys() if key in ['file', 'path']): + + if hasattr(path, 'keys') and any( + key for key in path.keys() if key in ['file', 'path'] + ): path = urlparse(path['file']).path if 'file' in path else path['path'] if not isinstance(path, six.string_types) or path == '*': return False + # If the string starts with a valid specifier operator, test if it is a valid # specifier set before making a path object (to avoid breaking windows) if any(path.startswith(spec) for spec in '!=<>~'): @@ -760,14 +759,18 @@ def is_installable_file(path): pass else: return False + if not os.path.exists(os.path.abspath(path)): return False + lookup_path = Path(path) absolute_path = '{0}'.format(lookup_path.absolute()) if lookup_path.is_dir() and is_installable_dir(absolute_path): return True + elif lookup_path.is_file() and is_archive_file(absolute_path): return True + return False @@ -797,23 +800,26 @@ def pep440_version(version): def pep423_name(name): """Normalize package name to PEP 423 style standard.""" name = name.lower() - if any(i not in name for i in (VCS_LIST+SCHEME_LIST)): + if any(i not in name for i in (VCS_LIST + SCHEME_LIST)): return name.replace('_', '-') + else: return name def proper_case(package_name): """Properly case project name from pypi.org.""" - # Hit the simple API. - r = requests.get('https://pypi.org/pypi/{0}/json'.format(package_name), timeout=0.3, stream=True) + r = requests.get( + 'https://pypi.org/pypi/{0}/json'.format(package_name), timeout=0.3, stream=True + ) if not r.ok: - raise IOError('Unable to find package {0} in PyPI repository.'.format(package_name)) + raise IOError( + 'Unable to find package {0} in PyPI repository.'.format(package_name) + ) r = parse.parse('https://pypi.org/pypi/{name}/json', r.url) good_name = r['name'] - return good_name @@ -867,15 +873,22 @@ def split_section(input_file, section_suffix, test_function): def split_file(file_dict): """Split VCS and editable dependencies out from file.""" sections = { - 'vcs': is_vcs, - 'editable': lambda x: hasattr(x, 'keys') and x.get('editable') + 'vcs': is_vcs, 'editable': lambda x: hasattr(x, 'keys') and x.get('editable') } for k, func in sections.items(): file_dict = split_section(file_dict, k, func) return file_dict -def merge_deps(file_dict, project, dev=False, requirements=False, ignore_hashes=False, blocking=False, only=False): +def merge_deps( + file_dict, + project, + dev=False, + requirements=False, + ignore_hashes=False, + blocking=False, + only=False, +): """ Given a file_dict, merges dependencies and converts them to pip dependency lists. :param dict file_dict: The result of calling :func:`pipenv.utils.split_file` @@ -889,12 +902,18 @@ def merge_deps(file_dict, project, dev=False, requirements=False, ignore_hashes= """ deps = [] requirements_deps = [] - for section in list(file_dict.keys()): # Turn develop-vcs into ['develop', 'vcs'] - section_name, suffix = section.rsplit('-', 1) if '-' in section and not section == 'dev-packages' else (section, None) - if not file_dict[section] or section_name not in ('dev-packages', 'packages', 'default', 'develop'): + section_name, suffix = section.rsplit( + '-', 1 + ) if '-' in section and not section == 'dev-packages' else ( + section, None + ) + if not file_dict[section] or section_name not in ( + 'dev-packages', 'packages', 'default', 'develop' + ): continue + is_dev = section_name in ('dev-packages', 'develop') if is_dev and not dev: continue @@ -903,12 +922,13 @@ def merge_deps(file_dict, project, dev=False, requirements=False, ignore_hashes= for k, v in file_dict[section]: if 'hash' in v: del v['hash'] - # Block and ignore hashes for all suffixed sections (vcs/editable) no_hashes = True if suffix else ignore_hashes block = True if suffix else blocking include_index = True if not suffix else False - converted = convert_deps_to_pip(file_dict[section], project, r=False, include_index=include_index) + converted = convert_deps_to_pip( + file_dict[section], project, r=False, include_index=include_index + ) deps.extend((d, no_hashes, block) for d in converted) if dev and is_dev and requirements: requirements_deps.extend((d, no_hashes, block) for d in converted) @@ -917,15 +937,12 @@ def merge_deps(file_dict, project, dev=False, requirements=False, ignore_hashes= def recase_file(file_dict): """Recase file before writing to output.""" - if 'packages' in file_dict or 'dev-packages' in file_dict: sections = ('packages', 'dev-packages') elif 'default' in file_dict or 'develop' in file_dict: sections = ('default', 'develop') - for section in sections: file_section = file_dict.get(section, {}) - # Try to properly case each key if we can. for key in list(file_section.keys()): try: @@ -933,7 +950,6 @@ def recase_file(file_dict): except IOError: cased_key = key file_section[cased_key] = file_section.pop(key) - return file_dict @@ -957,10 +973,11 @@ def find_windows_executable(bin_path, exe_name): exec_files = [filename for filename in exec_paths if os.path.isfile(filename)] if exec_files: return exec_files[0] + return find_executable(exe_name) -def get_converted_relative_path(path, relative_to=os.curdir): +def get_converted_relative_path(path, relative_to= os.curdir): """Given a vague relative path, return the path relative to the given location""" return os.path.join('.', os.path.relpath(path, start=relative_to)) @@ -969,9 +986,7 @@ def walk_up(bottom): """Mimic os.walk, but walk 'up' instead of down the directory tree. From: https://gist.github.com/zdavkeos/1098474 """ - bottom = os.path.realpath(bottom) - # Get files in current dir. try: names = os.listdir(bottom) @@ -984,11 +999,9 @@ def walk_up(bottom): dirs.append(name) else: nondirs.append(name) - yield bottom, dirs, nondirs new_path = os.path.realpath(os.path.join(bottom, '..')) - # See if we are at the top. if new_path == bottom: return @@ -999,19 +1012,20 @@ def walk_up(bottom): def find_requirements(max_depth=3): """Returns the path of a Pipfile in parent directories.""" - i = 0 for c, d, f in walk_up(os.getcwd()): i += 1 - if i < max_depth: if 'requirements.txt': r = os.path.join(c, 'requirements.txt') if os.path.isfile(r): return r + raise RuntimeError('No requirements.txt found!') + + # Borrowed from pew to avoid importing pew which imports psutil # See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 @contextmanager @@ -1020,6 +1034,7 @@ def temp_environ(): environ = dict(os.environ) try: yield + finally: os.environ.clear() os.environ.update(environ) @@ -1047,9 +1062,11 @@ def need_update_check(): p = os.sep.join((PIPENV_CACHE_DIR, '.pipenv_update_check')) if not os.path.exists(p): return True + out_of_date_time = time() - (24 * 60 * 60) if os.path.isfile(p) and os.path.getmtime(p) <= out_of_date_time: return True + else: return False @@ -1076,10 +1093,12 @@ def normalize_drive(path): """ if os.name != 'nt' or not isinstance(path, six.string_types): return path + drive, tail = os.path.splitdrive(path) # Only match (lower cased) local drives (e.g. 'c:'), not UNC mounts. if drive.islower() and len(drive) == 2 and drive[1] == ':': return '{}{}'.format(drive.upper(), tail) + return path @@ -1090,6 +1109,7 @@ def is_readonly_path(fn): """ if os.path.exists(fn): return (os.stat(fn).st_mode & stat.S_IREAD) or not os.access(fn, os.W_OK) + return False @@ -1100,7 +1120,9 @@ def set_write_bit(fn): def rmtree(directory, ignore_errors=False): - shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly) + shutil.rmtree( + directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly + ) def handle_remove_readonly(func, path, exc): @@ -1121,9 +1143,11 @@ def handle_remove_readonly(func, path, exc): if e.errno in [errno.EACCES, errno.EPERM]: warnings.warn(default_warning_message.format(path), ResourceWarning) return + if exc_exception.errno in [errno.EACCES, errno.EPERM]: warnings.warn(default_warning_message.format(path), ResourceWarning) return + raise @@ -1142,17 +1166,19 @@ class TemporaryDirectory(object): def __init__(self, suffix=None, prefix=None, dir=None): if 'RAM_DISK' in os.environ: import uuid + name = uuid.uuid4().hex dir_name = os.path.sep.join([os.environ['RAM_DISK'].strip(), name]) os.mkdir(dir_name) self.name = dir_name - else: self.name = tempfile.mkdtemp(suffix, prefix, dir) - self._finalizer = finalize( - self, self._cleanup, self.name, - warn_message="Implicitly cleaning up {!r}".format(self)) + self, + self._cleanup, + self.name, + warn_message="Implicitly cleaning up {!r}".format(self), + ) @classmethod def _cleanup(cls, name, warn_message):