From ad1c8cb67f0e1021bc05f295a9aff50a96dbe56a Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Mon, 4 Apr 2022 18:32:51 +0200 Subject: [PATCH 1/6] change subclassing behavior to Python standard --- upath/core.py | 40 +++++-------------------- upath/tests/implementations/test_gcs.py | 2 +- upath/tests/test_core.py | 10 ++++++- 3 files changed, 17 insertions(+), 35 deletions(-) diff --git a/upath/core.py b/upath/core.py index 90ef6d8e..1ae2f5d0 100644 --- a/upath/core.py +++ b/upath/core.py @@ -1,8 +1,6 @@ -import os import pathlib import re import urllib -from abc import ABCMeta from fsspec.registry import ( get_filesystem_class, @@ -87,21 +85,9 @@ def __getattribute__(self, item): ) -class PureUPath(pathlib.PurePath): - _flavour = pathlib._posix_flavour - __slots__ = () - - -class UPathMeta(ABCMeta): - def __instancecheck__(cls, instance): - return isinstance(instance, pathlib.Path) - - def __subclasscheck__(cls, subclass): - return issubclass(subclass, pathlib.Path) - - -class UPath(pathlib.Path, PureUPath, metaclass=UPathMeta): +class UPath(pathlib.Path): + _flavour = pathlib._posix_flavour __slots__ = ("_url", "_kwargs", "_closed", "fs") not_implemented = [ @@ -132,21 +118,8 @@ def __new__(cls, *args, **kwargs): val = kwargs.get(key) if val: parsed_url = parsed_url._replace(**{key: val}) - # treat as local filesystem, return PosixPath or WindowsPath impls = list(registry) + list(known_implementations.keys()) - if not parsed_url.scheme or parsed_url.scheme not in impls: - cls = ( - pathlib.WindowsPath - if os.name == "nt" - else pathlib.PosixPath - ) - self = cls._from_parts(args, init=False) - if not self._flavour.is_supported: - raise NotImplementedError( - "cannot instantiate %r on your system" % (cls.__name__,) - ) - self._init() - else: + if parsed_url.scheme and parsed_url.scheme in impls: import upath.registry cls = upath.registry._registry[parsed_url.scheme] @@ -155,9 +128,10 @@ def __new__(cls, *args, **kwargs): args = tuple(args_list) self = cls._from_parts_init(args, init=False) self._init(*args, **kwargs) - else: - self = super().__new__(*args, **kwargs) - return self + return self + + # treat as local filesystem, return PosixPath or WindowsPath + return pathlib.Path(*args, **kwargs) def _init(self, *args, template=None, **kwargs): self._closed = False diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py index 0e868650..f99a2ec1 100644 --- a/upath/tests/implementations/test_gcs.py +++ b/upath/tests/implementations/test_gcs.py @@ -7,7 +7,7 @@ from upath.tests.cases import BaseTests -@pytest.mark.skipif(sys.platform.startswith("win"), reason="Windows bad") +@pytest.mark.skipif(not sys.platform == "linux", reason="Only linux") @pytest.mark.usefixtures("path") class TestGCSPath(BaseTests): @pytest.fixture(autouse=True, scope="function") diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 7a9b7e05..25253566 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -81,11 +81,18 @@ class MyPath(UPath): assert isinstance(path, pathlib.Path) +def test_subclass_with_gcs(): + path = UPath("gcs://bucket", anon=True) + assert isinstance(path, UPath) + assert isinstance(path, pathlib.Path) + + def test_instance_check(local_testdir): path = pathlib.Path(local_testdir) upath = UPath(local_testdir) # test instance check passes - assert isinstance(upath, UPath) + assert isinstance(upath, pathlib.Path) + assert not isinstance(upath, UPath) # test type is same as pathlib assert type(upath) is type(path) upath = UPath(f"file://{local_testdir}") @@ -97,6 +104,7 @@ def test_new_method(local_testdir): path = UPath.__new__(pathlib.Path, local_testdir) assert str(path) == str(pathlib.Path(local_testdir)) assert isinstance(path, pathlib.Path) + assert not isinstance(path, UPath) @pytest.mark.skipif( From 0bdf0330ad95e645b2b68a338ba6aa7f5322498d Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Wed, 6 Apr 2022 21:56:49 +0200 Subject: [PATCH 2/6] fixes --- upath/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/upath/core.py b/upath/core.py index 4f8c388f..946848dc 100644 --- a/upath/core.py +++ b/upath/core.py @@ -119,9 +119,9 @@ def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]: val = kwargs.get(key) if val: parsed_url = parsed_url._replace(**{key: val}) - # treat as local filesystem, return PosixPath or WindowsPath - impls = list(registry) + list(known_implementations.keys()) - if parsed_url.scheme and parsed_url.scheme in impls: + + fsspec_impls = list(registry) + list(known_implementations.keys()) + if parsed_url.scheme and parsed_url.scheme in fsspec_impls: import upath.registry cls = upath.registry._registry[parsed_url.scheme] @@ -129,6 +129,7 @@ def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]: args_list.insert(0, parsed_url.path) args = tuple(args_list) return cls._from_parts(args, **kwargs) + # treat as local filesystem, return PosixPath or WindowsPath return pathlib.Path(*args, **kwargs) def __getattr__(self, item): From eae14c202f13ad9b15bddee3e1f673ce90c837ce Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Wed, 6 Apr 2022 22:01:44 +0200 Subject: [PATCH 3/6] lint --- upath/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upath/core.py b/upath/core.py index 946848dc..cf3f884e 100644 --- a/upath/core.py +++ b/upath/core.py @@ -119,7 +119,7 @@ def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]: val = kwargs.get(key) if val: parsed_url = parsed_url._replace(**{key: val}) - + fsspec_impls = list(registry) + list(known_implementations.keys()) if parsed_url.scheme and parsed_url.scheme in fsspec_impls: import upath.registry From 479d1eda2b34b4f8bc70964e5226f9cf206325a0 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Thu, 7 Apr 2022 20:51:11 +0200 Subject: [PATCH 4/6] merge --- upath/core.py | 67 +++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/upath/core.py b/upath/core.py index 55cbac53..95d1e7f8 100644 --- a/upath/core.py +++ b/upath/core.py @@ -104,43 +104,42 @@ class UPath(pathlib.Path): ] def __new__(cls, *args, **kwargs) -> Union["UPath", pathlib.Path]: - if issubclass(cls, UPath): - args_list = list(args) - first = args_list.pop(0) - if isinstance(first, pathlib.PurePath): - # Create a (modified) copy, if first arg is a Path object - other = first - parts = args_list - drv, root, parts = other._parse_args(parts) - drv, root, parts = other._flavour.join_parsed_parts( - other._drv, other._root, other._parts, drv, root, parts - ) + args_list = list(args) + first = args_list.pop(0) + if isinstance(first, pathlib.PurePath): + # Create a (modified) copy, if first arg is a Path object + other = first + parts = args_list + drv, root, parts = other._parse_args(parts) + drv, root, parts = other._flavour.join_parsed_parts( + other._drv, other._root, other._parts, drv, root, parts + ) - new_kwargs = getattr(other, "_kwargs", {}).copy() - new_kwargs.pop("_url", None) - new_kwargs.update(kwargs) + new_kwargs = getattr(other, "_kwargs", {}).copy() + new_kwargs.pop("_url", None) + new_kwargs.update(kwargs) - return other.__class__( - other._format_parsed_parts(drv, root, parts), - **new_kwargs, - ) + return other.__class__( + other._format_parsed_parts(drv, root, parts), + **new_kwargs, + ) + + url = stringify_path(first) + parsed_url = urllib.parse.urlparse(url) + for key in ["scheme", "netloc"]: + val = kwargs.get(key) + if val: + parsed_url = parsed_url._replace(**{key: val}) + + fsspec_impls = list(registry) + list(known_implementations.keys()) + if parsed_url.scheme and parsed_url.scheme in fsspec_impls: + import upath.registry + + cls = upath.registry._registry[parsed_url.scheme] + kwargs["_url"] = parsed_url + args_list.insert(0, parsed_url.path) + return cls._from_parts(tuple(args_list), **kwargs) - url = stringify_path(first) - parsed_url = urllib.parse.urlparse(url) - for key in ["scheme", "netloc"]: - val = kwargs.get(key) - if val: - parsed_url = parsed_url._replace(**{key: val}) - - fsspec_impls = list(registry) + list(known_implementations.keys()) - if parsed_url.scheme and parsed_url.scheme in fsspec_impls: - import upath.registry - - cls = upath.registry._registry[parsed_url.scheme] - kwargs["_url"] = parsed_url - args_list.insert(0, parsed_url.path) - args = tuple(args_list) - return cls._from_parts(args, **kwargs) # treat as local filesystem, return PosixPath or WindowsPath return pathlib.Path(*args, **kwargs) From 0d7c35ed8f15b7dd7f36255154e29cfc10c88b57 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Thu, 7 Apr 2022 21:41:31 +0200 Subject: [PATCH 5/6] tests --- upath/tests/test_core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 18c7b014..fa62b841 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -224,6 +224,11 @@ def test_copy_path_append(): assert str(path / "folder2" / "folder3") == str(copy_path) + path = UPath("/tmp/folder") + copy_path = UPath(path, "folder2", "folder3") + + assert str(path / "folder2" / "folder3") == str(copy_path) + def test_copy_path_append_kwargs(): path = UPath("gcs://bucket/folder", anon=True) From 744a8c293654b4bfc5d921ca83a2a4137f29ba9a Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Tue, 12 Apr 2022 09:55:54 +0200 Subject: [PATCH 6/6] merge --- upath/tests/implementations/test_gcs.py | 2 +- upath/tests/test_core.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py index f99a2ec1..0e868650 100644 --- a/upath/tests/implementations/test_gcs.py +++ b/upath/tests/implementations/test_gcs.py @@ -7,7 +7,7 @@ from upath.tests.cases import BaseTests -@pytest.mark.skipif(not sys.platform == "linux", reason="Only linux") +@pytest.mark.skipif(sys.platform.startswith("win"), reason="Windows bad") @pytest.mark.usefixtures("path") class TestGCSPath(BaseTests): @pytest.fixture(autouse=True, scope="function") diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index fa62b841..3c9296db 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -14,7 +14,6 @@ reason="don't run test on Windows", ) def test_posix_path(local_testdir): - print(type(UPath(local_testdir))) assert isinstance(UPath(local_testdir), pathlib.PosixPath) @@ -191,8 +190,6 @@ def test_copy_path(): path = UPath("gcs://bucket/folder", anon=True) copy_path = UPath(path) - print(type(path), type(copy_path)) - assert type(path) == type(copy_path) assert str(path) == str(copy_path) assert path._drv == copy_path._drv