Merge pull request #525 from FZJ-INM1-BDA/maint_sparseindex

Maint sparseindex

e2e/volumes/test_sparsemap_cache_uniqueness.py

@@ -1,7 +1,9 @@
 import siibra
+from siibra.volumes.sparsemap import SparseMap
 
 
 def test_sparsemap_cache_uniqueness():
     mp157 = siibra.get_map("julich 3.0", "colin 27", "statistical", spec="157")
     mp175 = siibra.get_map("julich 3.0", "colin 27", "statistical", spec="175")
+    assert isinstance(mp157, SparseMap) and isinstance(mp175, SparseMap)
     assert mp157.sparse_index.probs[0] != mp175.sparse_index.probs[0]

siibra/retrieval/repositories.py

@@ -19,7 +19,8 @@
     EbrainsRequest,
     SiibraHttpRequestError,
     find_suitiable_decoder,
-    DECODERS
+    DECODERS,
+    FileLoader
 )
 from .cache import CACHE
 
@@ -124,34 +125,16 @@ def __init__(self, folder: str):
         self._folder = pathlib.Path(folder)
         assert pathlib.Path.is_dir(self._folder)
 
-    def _build_url(self, folder: str, filename: str):
-        return pathlib.Path.joinpath(self._folder, folder, filename)
-
-    class FileLoader:
-        """
-        Just a loads a local file, but mimics the behaviour
-        of cached http requests used in other connectors.
-        """
-        def __init__(self, file_url, decode_func):
-            self.url = file_url
-            self.func = decode_func
-            self.cached = True
-
-        @property
-        def data(self):
-            with open(self.url, 'rb') as f:
-                return self.func(f.read())
+    def _build_url(self, folder: str, filename: str) -> str:
+        return pathlib.Path.joinpath(self._folder, folder, filename).as_posix()
 
     def get_loader(self, filename, folder="", decode_func=None):
         """Get a lazy loader for a file, for loading data
         only once loader.data is accessed."""
-        url = self._build_url(folder, filename)
-        if url is None:
-            raise RuntimeError(f"Cannot build url for ({folder}, {filename})")
-        if decode_func is None:
-            return self.FileLoader(url, lambda b: self._decode_response(b, filename))
-        else:
-            return self.FileLoader(url, decode_func)
+        filepath = self._build_url(folder, filename)
+        if not pathlib.Path(filepath).is_file():
+            raise RuntimeError(f"No file is found in {filepath}")
+        return FileLoader(filepath, decode_func)
 
     def search_files(self, folder="", suffix=None, recursive=False):
         results = []
@@ -165,7 +148,7 @@ def search_files(self, folder="", suffix=None, recursive=False):
     def __str__(self):
         return f"{self.__class__.__name__} at {self._folder}"
 
-    def __eq__(self, other):
+    def __eq__(self, other: "LocalFileRepository"):
         return self._folder == other._folder
 
 

siibra/retrieval/requests.py

@@ -117,8 +117,7 @@ def find_suitiable_decoder(url: str) -> Callable:
     suitable_decoders = [
         dec for sfx, dec in DECODERS.items() if urlpath.endswith(sfx)
     ]
-    if len(suitable_decoders) > 0:
-        assert len(suitable_decoders) == 1
+    if len(suitable_decoders) == 1:
         return suitable_decoders[0]
     else:
         return None
@@ -270,6 +269,24 @@ def data(self):
         return self.get()
 
 
+class FileLoader(HttpRequest):
+    """
+    Just a loads a local file, but mimics the behaviour
+    of cached http requests used in other connectors.
+    """
+    def __init__(self, filepath, func=None):
+        HttpRequest.__init__(
+            self, filepath, refresh=False,
+            func=func or find_suitiable_decoder(filepath)
+        )
+        self.cachefile = filepath
+
+    def _retrieve(self, **kwargs):
+        if kwargs:
+            logger.info(f"Keywords {list(kwargs.keys())} are supplied but won't be used.")
+        assert os.path.isfile(self.cachefile)
+
+
 class ZipfileRequest(HttpRequest):
     def __init__(self, url, filename, func=None, refresh=False):
         HttpRequest.__init__(