From 19640af9b7b6f2cc8630e09760c7951f797520f4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 21:02:18 -0400 Subject: [PATCH 1/4] Update vendored dependencies - Updated requirementslib - Fix local relative path bug whici caused paths to be output as absolute (test added to requirementslib) - Fix SSH normalization bug (test added to requirementslib) - Fixes #2440, #2441 Signed-off-by: Dan Ryan --- news/2453.bugfix | 2 + news/2453.vendor | 1 + pipenv/patched/piptools/utils.py | 2 +- pipenv/vendor/requirementslib/__init__.py | 2 +- .../requirementslib/models/requirements.py | 22 +++++++---- pipenv/vendor/requirementslib/models/utils.py | 2 +- pipenv/vendor/shellingham/__init__.py | 2 +- pipenv/vendor/shellingham/posix/__init__.py | 39 +++++++++++++++++++ pipenv/vendor/shellingham/posix/_default.py | 27 +++++++++++++ pipenv/vendor/shellingham/posix/linux.py | 35 +++++++++++++++++ pipenv/vendor/vendor.txt | 2 +- tasks/vendoring/__init__.py | 2 +- 12 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 news/2453.bugfix create mode 100644 news/2453.vendor create mode 100644 pipenv/vendor/shellingham/posix/__init__.py create mode 100644 pipenv/vendor/shellingham/posix/_default.py create mode 100644 pipenv/vendor/shellingham/posix/linux.py diff --git a/news/2453.bugfix b/news/2453.bugfix new file mode 100644 index 0000000000..7ea4d825cc --- /dev/null +++ b/news/2453.bugfix @@ -0,0 +1,2 @@ +Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. +Fixed a bug affecting normalization of ``git+git@host`` uris. diff --git a/news/2453.vendor b/news/2453.vendor new file mode 100644 index 0000000000..cea3b69517 --- /dev/null +++ b/news/2453.vendor @@ -0,0 +1 @@ +Updated ``requirementslib`` to version ``1.0.8`` diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 12c3991a81..66bfafa64f 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -11,7 +11,7 @@ from ._compat import InstallRequirement from first import first -from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier from .click import style diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index ae39b709f3..88623173e9 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -__version__ = "1.0.7" +__version__ = "1.0.8" from .exceptions import RequirementError diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index bd70001242..99d8ec76dd 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -173,6 +173,11 @@ def get_link_from_line(cls, line): # This is an URI. We'll need to perform some elaborated parsing. parsed_url = urllib_parse.urlsplit(fixed_line) + if added_ssh_scheme and ':' in parsed_url.netloc: + original_netloc, original_path_start = parsed_url.netloc.rsplit(':', 1) + uri_path = '/{0}{1}'.format(original_path_start, parsed_url.path) + original_url = parsed_url + parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) # Split the VCS part out if needed. original_scheme = parsed_url.scheme @@ -203,7 +208,8 @@ def get_link_from_line(cls, line): ) if added_ssh_scheme: - uri = strip_ssh_from_git_uri(uri) + original_uri = urllib_parse.urlunsplit(original_url._replace(scheme=original_scheme, fragment="")) + uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = Link( @@ -389,14 +395,14 @@ def from_pipfile(cls, name, pipfile): @property def line_part(self): - if ( + if self._uri_scheme and self._uri_scheme == 'path': + seed = self.path or unquote(self.link.url_without_fragment) or self.uri + elif ( (self._uri_scheme and self._uri_scheme == "file") - or (self.link.is_artifact or self.link.is_wheel) - and self.link.url + or ((self.link.is_artifact or self.link.is_wheel) + and self.link.url) ): seed = unquote(self.link.url_without_fragment) or self.uri - else: - seed = self.formatted_path or unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) if not self._has_hashed_name and self.is_remote_artifact: seed += "#egg={0}".format(self.name) @@ -528,8 +534,8 @@ def get_requirement(self): and "git+ssh://" in self.link.url and "git+git@" in self.uri ): - req.line = strip_ssh_from_git_uri(req.line) - req.uri = strip_ssh_from_git_uri(req.uri) + req.line = self.uri + req.uri = self.uri if not req.name: raise ValueError( "pipenv requires an #egg fragment for version controlled " diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index b7daf890b9..13b2b3683f 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -84,7 +84,7 @@ def strip_ssh_from_git_uri(uri): def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" + """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 64259c3a31..bbdab995a3 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -2,7 +2,7 @@ import os -__version__ = '1.0.0.dev1' +__version__ = '1.1.0' class ShellDetectionFailure(EnvironmentError): diff --git a/pipenv/vendor/shellingham/posix/__init__.py b/pipenv/vendor/shellingham/posix/__init__.py new file mode 100644 index 0000000000..ec27b3ac3f --- /dev/null +++ b/pipenv/vendor/shellingham/posix/__init__.py @@ -0,0 +1,39 @@ +import os +import platform + +from .._consts import SHELL_NAMES + + +def _get_process_mapping(): + system = platform.system() + if system == 'Linux': + from . import linux as impl + else: + from . import _default as impl + return impl.get_process_mapping() + + +def get_shell(pid=None, max_depth=6): + """Get the shell that the supplied pid or os.getpid() is running in. + """ + pid = str(pid or os.getpid()) + mapping = _get_process_mapping() + login_shell = os.environ.get('SHELL', '') + for _ in range(max_depth): + try: + proc = mapping[pid] + except KeyError: + break + name = os.path.basename(proc.args[0]).lower() + if name in SHELL_NAMES: + return (name, proc.args[0]) + elif proc.args[0].startswith('-'): + # This is the login shell. Use the SHELL environ if possible + # because it provides better information. + if login_shell: + name = login_shell.lower() + else: + name = proc.args[0][1:].lower() + return (os.path.basename(name), name) + pid = proc.ppid # Go up one level. + return None diff --git a/pipenv/vendor/shellingham/posix/_default.py b/pipenv/vendor/shellingham/posix/_default.py new file mode 100644 index 0000000000..8694427611 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/_default.py @@ -0,0 +1,27 @@ +import collections +import shlex +import subprocess +import sys + + +Process = collections.namedtuple('Process', 'args pid ppid') + + +def get_process_mapping(): + """Try to look up the process tree via the output of `ps`. + """ + output = subprocess.check_output([ + 'ps', '-ww', '-o', 'pid=', '-o', 'ppid=', '-o', 'args=', + ]) + if not isinstance(output, str): + output = output.decode(sys.stdout.encoding) + processes = {} + for line in output.split('\n'): + try: + pid, ppid, args = line.strip().split(None, 2) + except ValueError: + continue + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) + return processes diff --git a/pipenv/vendor/shellingham/posix/linux.py b/pipenv/vendor/shellingham/posix/linux.py new file mode 100644 index 0000000000..6db9783481 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/linux.py @@ -0,0 +1,35 @@ +import os +import re + +from ._default import Process + + +STAT_PPID = 3 +STAT_TTY = 6 + + +def get_process_mapping(): + """Try to look up the process tree via Linux's /proc + """ + with open('/proc/{0}/stat'.format(os.getpid())) as f: + self_tty = f.read().split()[STAT_TTY] + processes = {} + for pid in os.listdir('/proc'): + if not pid.isdigit(): + continue + try: + stat = '/proc/{0}/stat'.format(pid) + cmdline = '/proc/{0}/cmdline'.format(pid) + with open(stat) as fstat, open(cmdline) as fcmdline: + stat = re.findall(r'\(.+\)|\S+', fstat.read()) + cmd = fcmdline.read().split('\x00')[:-1] + ppid = stat[STAT_PPID] + tty = stat[STAT_TTY] + if tty == self_tty: + processes[pid] = Process( + args=tuple(cmd), pid=pid, ppid=ppid, + ) + except IOError: + # Process has disappeared - just ignore it. + continue + return processes diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index e95b517840..8ecc6ba5f3 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -34,7 +34,7 @@ requirementslib==1.0.7 pyparsing==2.2.0 pytoml==0.1.16 requirements-parser==0.2.0 -shellingham==1.1.0dev0 +shellingham==1.1.0 six==1.11.0 semver==2.8.0 shutilwhich==1.1.0 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index f0e4eddf9c..d61d765527 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -453,7 +453,7 @@ def license_destination(vendor_dir, libname, filename): normal = vendor_dir / libname if normal.is_dir(): return normal / filename - lowercase = vendor_dir / libname.lower() + lowercase = vendor_dir / libname.lower().replace('-', '_') if lowercase.is_dir(): return lowercase / filename rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES From 3be2badf1f1b58c82ac99557312659850d0baefe Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 27 Jun 2018 21:06:32 -0400 Subject: [PATCH 2/4] Update vendored requirementslib version Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 8ecc6ba5f3..0b2808f010 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -27,7 +27,7 @@ requests==2.19.1 idna==2.7 urllib3==1.23 certifi==2018.4.16 -requirementslib==1.0.7 +requirementslib==1.0.8 attrs==18.1.0 distlib==0.2.7 packaging==17.1 From 1ed07ff0b944b44b743804273384dea0365696c5 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 28 Jun 2018 15:40:43 -0400 Subject: [PATCH 3/4] Update pipfile and setup.py Signed-off-by: Dan Ryan --- Pipfile | 1 + Pipfile.lock | 11 ++++++++++- setup.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index ec959c3261..f7a0743769 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ black = {version="*", markers="python_version >= '3.6'"} pytz = "*" towncrier = {git = "https://github.com/hawkowl/towncrier.git", editable = true, ref = "master"} parver = "*" +invoke = "*" [packages] diff --git a/Pipfile.lock b/Pipfile.lock index 4503fff0b1..48d2873df1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -185,6 +185,15 @@ ], "version": "==17.5.0" }, + "invoke": { + "hashes": [ + "sha256:21274204515dca62206470b088bbcf9d41ffda82b3715b90e01d71b7a4681921", + "sha256:4a4cc031db311cbfb3fdd8ec93a06c892533c27b931f4be14b24c97cd042b14e", + "sha256:621b6564f992c37166e16090d7e7cccb3b922e03a58e980dfa5e543a931b652f" + ], + "index": "pypi", + "version": "==1.0.0" + }, "itsdangerous": { "hashes": [ "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" @@ -480,4 +489,4 @@ "version": "==0.14.1" } } -} +} \ No newline at end of file diff --git a/setup.py b/setup.py index 2cbe33ff70..a2a29c3efe 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,7 @@ def run(self): "pipenv.patched.notpip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"], }, python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*', + setup_requires=['invoke', 'parver'], install_requires=required, extras_require={}, include_package_data=True, From b33356012841ad9c41878cf29a75e58307b336e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 28 Jun 2018 16:38:08 -0400 Subject: [PATCH 4/4] Update lockfile Signed-off-by: Dan Ryan --- Pipfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 48d2873df1..21b3270544 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ee9e8c04f9a6b7ad4cdaa7db728b51b9fef04b6cad9f012edd230a6241b43f92" + "sha256": "0599ae763bc5f1eaa30bd414c1495dc10fdfd3b5b0c109515d71322c8668d7e1" }, "pipfile-spec": 6, "requires": {}, @@ -489,4 +489,4 @@ "version": "==0.14.1" } } -} \ No newline at end of file +}