From 42bb8ac3bc0ff8b0a43542a595e5afeb473710b9 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 16 Nov 2023 16:02:39 -0600 Subject: [PATCH 1/3] test different handlers --- tests/test_0302_pickle.py | 42 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/test_0302_pickle.py b/tests/test_0302_pickle.py index 1177c7088..3f13afe81 100644 --- a/tests/test_0302_pickle.py +++ b/tests/test_0302_pickle.py @@ -9,11 +9,18 @@ import uproot -pytest.importorskip("awkward") - -def test_pickle_roundtrip_mmap(): - with uproot.open(skhep_testdata.data_path("uproot-small-dy-withoffsets.root")) as f: +@pytest.mark.parametrize( + "handler", + [ + uproot.source.file.MemmapSource, + uproot.source.fsspec.FSSpecSource, + ], +) +def test_pickle_roundtrip_local(handler): + with uproot.open( + skhep_testdata.data_path("uproot-small-dy-withoffsets.root"), handler=handler + ) as f: pkl = pickle.dumps(f["tree"]) branch = pickle.loads(pkl)["Muon_pt"] @@ -32,9 +39,20 @@ def test_pickle_roundtrip_mmap(): ] +@pytest.mark.parametrize( + "handler", + [ + uproot.source.http.HTTPSource, + uproot.source.fsspec.FSSpecSource, + ], +) @pytest.mark.network -def test_pickle_roundtrip_http(): - with uproot.open("https://scikit-hep.org/uproot3/examples/Zmumu.root") as f: +def test_pickle_roundtrip_http(handler): + pytest.importorskip("aiohttp") + + with uproot.open( + "https://scikit-hep.org/uproot3/examples/Zmumu.root", handler=handler + ) as f: pkl = pickle.dumps(f["events"]) tree = pickle.loads(pkl) @@ -53,12 +71,20 @@ def test_pickle_roundtrip_http(): ] +@pytest.mark.parametrize( + "handler", + [ + uproot.source.xrootd.XRootDSource, + uproot.source.fsspec.FSSpecSource, + ], +) @pytest.mark.network @pytest.mark.xrootd -def test_pickle_roundtrip_xrootd(): +def test_pickle_roundtrip_xrootd(handler): pytest.importorskip("XRootD") with uproot.open( - "root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/Run2012B_DoubleMuParked.root" + "root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/Run2012B_DoubleMuParked.root", + handler=handler, ) as f: pkl = pickle.dumps(f["Events"]) From 69f96d872f96e43c28e23ccd74ea209c85809d59 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 16 Nov 2023 17:06:31 -0600 Subject: [PATCH 2/3] correct serialization of fsspec source --- src/uproot/source/fsspec.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/uproot/source/fsspec.py b/src/uproot/source/fsspec.py index 2cb874575..5edf2d138 100644 --- a/src/uproot/source/fsspec.py +++ b/src/uproot/source/fsspec.py @@ -33,20 +33,27 @@ def __init__(self, file_path: str, **options): if k not in uproot.reading.open.defaults.keys() } - self._executor = FSSpecLoopExecutor() - self._fs, self._file_path = fsspec.core.url_to_fs(file_path, **storage_options) # What should we do when there is a chain of filesystems? self._async_impl = self._fs.async_impl - self._file = self._fs.open(self._file_path) + self._executor = None + self._file = None self._fh = None + self._num_requests = 0 self._num_requested_chunks = 0 self._num_requested_bytes = 0 + + self._open() + self.__enter__() + def _open(self): + self._executor = FSSpecLoopExecutor() + self._file = self._fs.open(self._file_path) + def __repr__(self): path = repr(self._file_path) if len(self._file_path) > 10: @@ -56,6 +63,8 @@ def __repr__(self): def __getstate__(self): state = dict(self.__dict__) state.pop("_executor") + state.pop("_file") + state.pop("_fh") return state def __setstate__(self, state): From 817688e4a049c6367a0b3d23c6fee2ae6947227b Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 16 Nov 2023 17:23:59 -0600 Subject: [PATCH 3/3] fsspec is not yet required --- tests/test_0302_pickle.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_0302_pickle.py b/tests/test_0302_pickle.py index 3f13afe81..fbd436511 100644 --- a/tests/test_0302_pickle.py +++ b/tests/test_0302_pickle.py @@ -1,8 +1,6 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE -import os import pickle -import sys import pytest import skhep_testdata @@ -14,7 +12,7 @@ "handler", [ uproot.source.file.MemmapSource, - uproot.source.fsspec.FSSpecSource, + # uproot.source.fsspec.FSSpecSource, ], ) def test_pickle_roundtrip_local(handler): @@ -43,7 +41,7 @@ def test_pickle_roundtrip_local(handler): "handler", [ uproot.source.http.HTTPSource, - uproot.source.fsspec.FSSpecSource, + # uproot.source.fsspec.FSSpecSource, ], ) @pytest.mark.network @@ -75,7 +73,7 @@ def test_pickle_roundtrip_http(handler): "handler", [ uproot.source.xrootd.XRootDSource, - uproot.source.fsspec.FSSpecSource, + # uproot.source.fsspec.FSSpecSource, ], ) @pytest.mark.network