From f0a1413c70847c6604f3b9a0a1944f9afa07b704 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Tue, 10 Jul 2018 20:41:34 -0500 Subject: [PATCH 1/7] [#2504] Add test for VCS dep with extras --- tests/integration/test_lock.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1ab34f1b92..eebfaf39e6 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -1,6 +1,5 @@ import pytest import os -import six from pipenv.utils import temp_environ @@ -348,6 +347,28 @@ def test_lock_editable_vcs_without_install(PipenvInstance, pypi): assert c.return_code == 0 +@pytest.mark.extras +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["security"]} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert 'requests' in p.lockfile['default'] + assert 'idna' in p.lockfile['default'] + assert 'chardet' in p.lockfile['default'] + assert 'cryptography' in p.lockfile['default'] + assert 'pyOpenSSL' in p.lockfile['default'] + c = p.pipenv('install') + assert c.return_code == 0 + + @pytest.mark.lock @pytest.mark.skip(reason="This doesn't work for some reason.") def test_lock_respecting_python_version(PipenvInstance, pypi): From 6e05a3e0d27182b21381455751fcff0ec57f7d53 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Tue, 10 Jul 2018 20:58:57 -0500 Subject: [PATCH 2/7] [#2504] Serialize prettytoml ArrayElements as lists --- pipenv/project.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index 7f40576d99..d0d21c3eca 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,6 +14,7 @@ import six import toml import json as simplejson +from prettytoml.elements.array import ArrayElement from ._compat import Path @@ -64,6 +65,12 @@ def _normalized(p): DEFAULT_NEWLINES = u"\n" +def encode_toml_elements(obj): + if isinstance(obj, ArrayElement): + return obj.primitive_value + raise TypeError(repr(obj) + " is not JSON serializable") + + def preferred_newlines(f): if isinstance(f.newlines, six.text_type): return f.newlines @@ -631,7 +638,8 @@ def write_lockfile(self, content): """ newlines = self._lockfile_newlines s = simplejson.dumps( # Send Unicode in to guarentee Unicode out. - content, indent=4, separators=(u",", u": "), sort_keys=True + content, indent=4, separators=(u",", u": "), sort_keys=True, + default=encode_toml_elements, ) with atomic_open_for_write(self.lockfile_location, newline=newlines) as f: f.write(s) From 6de915ab79f5e23383f510ef70b6b389a40e51e8 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 08:19:42 -0500 Subject: [PATCH 3/7] [#2504] Replace security extra with socks in test --- tests/integration/test_lock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index eebfaf39e6..67da46b9d9 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -356,15 +356,14 @@ def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["security"]} +requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 assert 'requests' in p.lockfile['default'] assert 'idna' in p.lockfile['default'] assert 'chardet' in p.lockfile['default'] - assert 'cryptography' in p.lockfile['default'] - assert 'pyOpenSSL' in p.lockfile['default'] + assert 'pysocks' in p.lockfile['default'] c = p.pipenv('install') assert c.return_code == 0 From 1ceccb9bdf20d7b54913a886a3b6273c40366e1c Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Wed, 11 Jul 2018 23:32:43 -0500 Subject: [PATCH 4/7] Fix extras check in test --- tests/integration/test_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 67da46b9d9..25195bfbec 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -363,7 +363,7 @@ def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): assert 'requests' in p.lockfile['default'] assert 'idna' in p.lockfile['default'] assert 'chardet' in p.lockfile['default'] - assert 'pysocks' in p.lockfile['default'] + assert "socks" in p.lockfile["default"]["requests"]["extras"] c = p.pipenv('install') assert c.return_code == 0 From dbc9008131f5c9684e6b5336a4ca2993f518d9a6 Mon Sep 17 00:00:00 2001 From: Jacob Hayes Date: Thu, 12 Jul 2018 08:05:58 -0500 Subject: [PATCH 5/7] Duck type prettytoml types during serialization --- pipenv/project.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index d0d21c3eca..918c7184c8 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -14,7 +14,6 @@ import six import toml import json as simplejson -from prettytoml.elements.array import ArrayElement from ._compat import Path @@ -66,7 +65,7 @@ def _normalized(p): def encode_toml_elements(obj): - if isinstance(obj, ArrayElement): + if hasattr(obj, 'primitive_value'): return obj.primitive_value raise TypeError(repr(obj) + " is not JSON serializable") From 98a52149d7a85419f3067fa3cba2fe1deb59e705 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Jul 2018 15:03:59 +0800 Subject: [PATCH 6/7] Seperate lock file concerns in a custom encoder --- pipenv/project.py | 51 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 918c7184c8..1dc2aaa449 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -13,7 +13,6 @@ import pipfile.api import six import toml -import json as simplejson from ._compat import Path @@ -64,16 +63,36 @@ def _normalized(p): DEFAULT_NEWLINES = u"\n" -def encode_toml_elements(obj): - if hasattr(obj, 'primitive_value'): - return obj.primitive_value - raise TypeError(repr(obj) + " is not JSON serializable") +class _LockFileEncoder(json.JSONEncoder): + """A specilized JSON encoder to convert loaded TOML data into a lock file. + + This adds a few characteristics to the encoder: + + * The JSON is always prettified with indents and spaces. + * PrettyTOML's container elements are seamlessly encodable. + * The output is always UTF-8-encoded text, never binary, even on Python 2. + """ + def __init__(self): + super(_LockFileEncoder, self).__init__( + indent=4, separators=(",", ": "), sort_keys=True, + ) + + def default(self, obj): + from prettytoml.elements.common import ContainerElement + if isinstance(obj, ContainerElement): + return obj.primitive_value + return super(_LockFileEncoder, self).default(obj) + + def encode(self, obj): + content = super(_LockFileEncoder, self).encode(obj) + if not isinstance(content, six.text_type): + content = content.decode("utf-8") + return content def preferred_newlines(f): if isinstance(f.newlines, six.text_type): return f.newlines - return DEFAULT_NEWLINES @@ -111,6 +130,8 @@ class SourceNotFound(KeyError): class Project(object): """docstring for Project""" + _lockfile_encoder = _LockFileEncoder() + def __init__(self, which=None, python_version=None, chdir=True): super(Project, self).__init__() self._name = None @@ -635,15 +656,17 @@ def write_toml(self, data, path=None): def write_lockfile(self, content): """Write out the lockfile. """ - newlines = self._lockfile_newlines - s = simplejson.dumps( # Send Unicode in to guarentee Unicode out. - content, indent=4, separators=(u",", u": "), sort_keys=True, - default=encode_toml_elements, - ) - with atomic_open_for_write(self.lockfile_location, newline=newlines) as f: + s = self._lockfile_encoder.encode(content) + open_kwargs = { + 'newline': self._lockfile_newlines, + 'encoding': 'utf-8', + } + with atomic_open_for_write(self.lockfile_location, **open_kwargs) as f: f.write(s) + # Write newline at end of document. GH-319. + # Only need '\n' here; the file object handles the rest. if not s.endswith(u"\n"): - f.write(u"\n") # Write newline at end of document. GH #319. + f.write(u"\n") @property def pipfile_sources(self): @@ -758,7 +781,7 @@ def recase_pipfile(self): self.write_toml(self.parsed_pipfile) def load_lockfile(self, expand_env_vars=True): - with io.open(self.lockfile_location) as lock: + with io.open(self.lockfile_location, encoding='utf-8') as lock: j = json.load(lock) self._lockfile_newlines = preferred_newlines(lock) # lockfile is just a string From 36239140017466baf2956fa1da2d640382d7ff82 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 13 Jul 2018 22:26:43 +0800 Subject: [PATCH 7/7] Also allow TokenElement in lock file encoder Can't be too careful hadling things here. --- pipenv/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/project.py b/pipenv/project.py index 1dc2aaa449..79fa1b91a0 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -78,8 +78,8 @@ def __init__(self): ) def default(self, obj): - from prettytoml.elements.common import ContainerElement - if isinstance(obj, ContainerElement): + from prettytoml.elements.common import ContainerElement, TokenElement + if isinstance(obj, (ContainerElement, TokenElement)): return obj.primitive_value return super(_LockFileEncoder, self).default(obj)