Chore/54 test coverage

CHANGELOG.md

@@ -14,6 +14,9 @@
  - standardise participation functions API and return columns
  - refactor `solvis.inversion_solution` package to `solvis.solution` and collect modules into packages
  - new packages `solvis.solution.inversion_solution` and  `solvis.solution.fault_system_solution`
+ - improved test coverage
+ - refactored to/from archive code to reginstate some skipped tests
+ - doc/typing improvements
 
 ## Added
  - new filter package providing classes for filtering solutions

setup.cfg

@@ -40,6 +40,7 @@ omit =
     solvis/dochelper/*
     solvis/get_secret.py
     solvis/config.py
+    solvis/solution/named_fault.py
 
 [coverage:report]
 exclude_lines =
@@ -52,6 +53,7 @@ exclude_lines =
 	if 0:
 	if __name__ == .__main__.:
 	def main
+    if TYPE_CHECKING:
 
 [tox:tox]
 isolated_build = true
@@ -74,8 +76,7 @@ setenv =
 	PYTHONPATH = {toxinidir}
 	PYTHONWARNINGS = ignore
 commands =
-	pytest -m "not performance and not slow" --cov=solvis --cov-branch --cov-report=xml --cov-report=term-missing test
-	pytest -m "performance or slow"
+	pytest --cov=solvis --cov-branch --cov-report=xml --cov-report=term-missing test
 
 [testenv:format]
 allowlist_externals =

solvis/fault_system_solution_helper.py

@@ -34,8 +34,7 @@ def build_rupture_groups(
             sample_len = len(sections)
             continue
 
-        if rupt_id is not sample_rupt:
-            sample_ruptures.append(rupt_id)
+        sample_ruptures.append(rupt_id)
 
         # compare section overlap
         section_overlap = len(set(sections).intersection(sample_sections))

solvis/filter/chainable_set_base.py

@@ -37,7 +37,7 @@ def new_chainable_set(
         elif join_prior == SetOperationEnum.DIFFERENCE:
             instance._chained_set = set.difference(result, self.chained_set) if self.chained_set else result
         else:
-            raise ValueError(f"Unsupported join type {join_prior}")
+            raise ValueError(f"Unsupported join type {join_prior}")  # pragma: no cover
         return instance
 
     def __eq__(self, other) -> bool:

solvis/filter/parent_fault_id_filter.py

@@ -99,7 +99,7 @@ def _model(self):
             return self.__model.model
         except (AttributeError):
             return self.__model
-        raise ValueError(f"unhandled type: {type(self.__model)}")
+        raise ValueError(f"unhandled type: {type(self.__model)}")  # pragma: no cover
 
     def for_named_faults(self, named_fault_names: Iterable[str]):
         raise NotImplementedError()

solvis/filter/rupture_id_filter.py

@@ -74,7 +74,7 @@ def _model(self):
             return self.__model.model
         except (AttributeError):
             return self.__model
-        raise ValueError(f"unhandled type: {type(self.__model)}")
+        raise ValueError(f"unhandled type: {type(self.__model)}")  # pragma: no cover
 
     def all(self) -> ChainableSetBase:
         """Convenience method returning ids for all solution ruptures.
@@ -285,7 +285,9 @@ def for_polygons(
         elif join_polygons == SetOperationEnum.DIFFERENCE:
             rupture_ids = set.difference(*rupture_id_sets)
         else:
-            raise ValueError("Only INTERSECTION, UNION & DIFFERENCE operations are supported for `join_type`")
+            raise ValueError(
+                "Only INTERSECTION, UNION & DIFFERENCE operations are supported for `join_type`"
+            )  # pragma: no cover
         return self.new_chainable_set(rupture_ids, self._model, self._drop_zero_rates, join_prior=join_prior)
 
     def for_polygon(

solvis/filter/subsection_id_filter.py

@@ -45,10 +45,10 @@ def _model(self):
             return self.__model.model
         except (AttributeError):
             return self.__model
-        raise ValueError(f"unhandled type: {type(self.__model)}")
+        raise ValueError(f"unhandled type: {type(self.__model)}")  # pragma: no cover
 
     def for_named_faults(self, named_fault_names: Iterable[str]) -> ChainableSetBase:
-        raise NotImplementedError()
+        raise NotImplementedError()  # pragma: no cover until named fault feature is implemented
 
     def all(self) -> ChainableSetBase:
         """Convenience method returning ids for all solution fault subsections.

solvis/solution/composite_solution.py

@@ -6,26 +6,21 @@
 """
 import io
 import logging
-import time
 import zipfile
 from pathlib import Path
 from typing import Any, Dict, Iterable, Optional, Union
 
 import geopandas as gpd
 import pandas as pd
 
-from .fault_system_solution import FaultSystemSolution
-
-# from .typing import CompositeSolutionProtocol
-from .inversion_solution.inversion_solution_model import CompositeSolutionModel
-
-# from .inversion_solution_file import data_to_zip_direct
+from solvis.solution.inversion_solution.inversion_solution_file import data_to_zip_direct
 
+from .fault_system_solution import FaultSystemSolution
 
 log = logging.getLogger(__name__)
 
 
-class CompositeSolution(CompositeSolutionModel):
+class CompositeSolution:
     """A container class collecting FaultSystemSolution instances and a source_logic_tree.
 
     Methods:
@@ -51,7 +46,9 @@ def add_fault_system_solution(self, fault_system: str, fault_system_solution: Fa
         """Add a new FaultSystemSolution instance."""
         # print(">>> add_fault_system_solution", self, fault_system)
         if fault_system in self._solutions.keys():
-            raise ValueError(f"fault system with key: {fault_system} exists already. {self._solutions.keys()}")
+            raise ValueError(
+                f"fault system with key: {fault_system} exists already. {self._solutions.keys()}"
+            )  # pragma: no cover
         self._solutions[fault_system] = fault_system_solution
         return self
 
@@ -148,23 +145,12 @@ def to_archive(self, archive_path: Union[Path, str]):
             for key, fss in self._solutions.items():
                 fss_name = f"{key}_fault_system_solution.zip"
                 fss_file = fss.solution_file
-                if fss_file.archive:
-                    # we can serialise the 'in-memory' archive now
-                    # data_to_zip_direct(zout, fss_file.archive, fss_name)
-
-                    # TODO : consider how to resolve this, it's needed from creating composite archive
-                    # and it was written to store fss archive to disk
-
-                    assert 0
-                    log.debug('direct store %s' % fss_name)
-                    zinfo = zipfile.ZipInfo(fss_name, time.localtime()[:6])
-                    zinfo.compress_type = zipfile.ZIP_DEFLATED
-                    zout.write(zinfo, fss_file.archive.read(), fss_name)
-
-                elif fss_file.archive_path is None:
-                    raise RuntimeError("archive_path is not defined")
-                else:
-                    zout.write(fss_file.archive_path, arcname=fss_name)
+                if fss_file._archive:
+                    # serialise the 'in-memory' archive
+                    fss_file._archive.seek(0)
+                    data_to_zip_direct(zout, fss_file._archive.read(), fss_name)
+                else:  # pragma: no cover
+                    raise RuntimeError("_archive is not defined")
         self._archive_path = Path(archive_path)
 
     @staticmethod

solvis/solution/fault_system_solution/fault_system_solution.py

@@ -69,6 +69,9 @@ def from_archive(instance_or_path: Union[Path, str, io.BytesIO]) -> 'FaultSystem
         # TODO: sort out this weirdness
         if isinstance(instance_or_path, io.BytesIO):
             with zipfile.ZipFile(instance_or_path, 'r') as zf:
+
+                print("namelist: ", zf.namelist())
+
                 assert 'composite_rates.csv' in zf.namelist()
                 assert 'aggregate_rates.csv' in zf.namelist()
                 assert 'ruptures/fast_indices.csv' in zf.namelist()
@@ -83,6 +86,21 @@ def from_archive(instance_or_path: Union[Path, str, io.BytesIO]) -> 'FaultSystem
 
     @staticmethod
     def filter_solution(solution: 'FaultSystemSolution', rupture_ids: Iterable) -> 'FaultSystemSolution':
+        """
+        Filter a FaultSystemSolution by a subset of its rupture IDs, returing a new FaultSystemSolution.
+
+        Note:
+         - this is an utility method primarily for producing test fixtures.
+         - this is not actually used, and maybe deprecated in a future release
+
+        Parameters:
+            solution: an fault system solution instance.
+            rupture_ids: a sequence of rupture ids.
+
+        Returns:
+            A new FaultSystemSolution containing data for the rupture IDs specified.
+        """
+
         solution = cast(FaultSystemSolution, solution)
         model = solution.model
         rr = model.ruptures
@@ -108,30 +126,33 @@ def filter_solution(solution: 'FaultSystemSolution', rupture_ids: Iterable) -> '
             solution.solution_file.fault_regime,
             average_slips,
         )
-        # ns._archive_path = None
-        #### new_solution_file.enable_fast_indices()
-        # copy the original archive, if it exists
-        # TODO: does the archive needs filtering applied?? see to_archive()
-        if solution.solution_file._archive:
-            new_archive = io.BytesIO()
-            with zipfile.ZipFile(new_archive, 'w') as new_zip:
-                # write the core files
-                with zipfile.ZipFile(solution.solution_file._archive, 'r') as zf:
-                    for item in zf.filelist:
-                        if item.filename in solution.solution_file.DATAFRAMES:
-                            continue
-                        if item.filename in solution.solution_file.OPENSHA_ONLY:  # drop bulky, opensha-only artefacts
-                            continue
-                        new_zip.writestr(item, zf.read(item.filename))
-                # write the modifies tables
-                new_solution_file._write_dataframes(new_zip, reindex=False)  # retain original rupture ids and structure
-            new_solution_file._archive = new_archive
-            new_solution_file._archive.seek(0)
+
+        # now copy data from the original archive
+        assert solution.solution_file._archive, "Assumed _archive is not available"
+
+        new_archive = io.BytesIO()
+        with zipfile.ZipFile(new_archive, 'w') as new_zip:
+            # write the core files
+            with zipfile.ZipFile(solution.solution_file._archive, 'r') as zf:
+                for item in zf.filelist:
+                    if item.filename in solution.solution_file.DATAFRAMES:
+                        log.debug(f'filter_solution() skipping copy of dataframe file: {item.filename}')
+                        continue
+                    if item.filename in solution.solution_file.OPENSHA_ONLY:  # drop bulky, opensha-only artefacts
+                        log.debug(f'filter_solution() skipping copy of opensha only file: {item.filename}')
+                        continue
+                    log.debug(f'filter_solution() copying {item.filename}')
+                    new_zip.writestr(item, zf.read(item.filename))
+
+            # write the modifies tables
+            new_solution_file._write_dataframes(new_zip, reindex=False)  # retain original rupture ids and structure
+        new_solution_file._archive = new_archive
+        new_solution_file._archive.seek(0)
         return FaultSystemSolution(new_solution_file)
 
     @staticmethod
     def new_solution(solution: BranchSolutionProtocol, composite_rates_df: pd.DataFrame) -> 'FaultSystemSolution':
-        # build a new composite solution, taking solution template properties, and composite_rates_df
+        # build a new fault system solution, taking solution template properties, and composite_rates_df
         composite_rates_df = composite_rates_df[composite_rates_df["Annual Rate"] > 0]
         composite_rates_df.insert(
             0,
@@ -170,7 +191,10 @@ def new_solution(solution: BranchSolutionProtocol, composite_rates_df: pd.DataFr
             solution.solution_file.average_slips.copy(),
         )
         fss_file._archive_path = solution.solution_file.archive_path
-        return FaultSystemSolution(fss_file)
+
+        new_fss = FaultSystemSolution(fss_file)
+        new_fss.to_archive(io.BytesIO(), solution.solution_file.archive_path)  # initialise the _archive
+        return new_fss
 
     @staticmethod
     def get_branch_inversion_solution_id(branch: ModelLogicTreeBranch) -> str:
@@ -187,9 +211,11 @@ def get_branch_inversion_solution_id(branch: ModelLogicTreeBranch) -> str:
                 if source.type == "inversion":
                     inversion_solution_id = source.inversion_id
                     break
+                else:  # pragma: no cover
+                    continue
             else:
-                raise Exception("Could not find inversion solution ID for branch solution")
-        else:
+                raise Exception("Could not find inversion solution ID for branch solution")  # pragma: no cover
+        else:  # pragma: no cover
             # Fall back to v1 behaviour
             inversion_solution_id = branch.inversion_solution_id
 
@@ -212,7 +238,6 @@ def from_branch_solutions(solutions: Iterable[BranchSolutionProtocol]) -> 'Fault
             solution_df.insert(0, 'fault_system', branch_solution.fault_system)
             composite_rates_df = pd.concat([composite_rates_df, solution_df], ignore_index=True)
 
-            # print('dims', composite_rates_df.shape, solution_df.shape)
         return FaultSystemSolution.new_solution(solution=branch_solution, composite_rates_df=composite_rates_df)
 
 

solvis/solution/fault_system_solution/fault_system_solution_model.py

@@ -49,7 +49,7 @@ def rupture_sections(self) -> 'DataFrame[RuptureSectionSchema]':
         if self._fast_indices is None:
             try:
                 self._fast_indices = self._solution_file.fast_indices
-                log.debug("loaded fast indices")
+                log.debug("loaded fast indices")  # pragma: no cover
             except Exception:
                 log.info("rupture_sections() building fast indices")
                 self._fast_indices = super().build_rupture_sections()