Add examples

src/cpdbench/control/ValidationRunController.py

@@ -60,5 +60,5 @@ def execute_run(self, methods: dict) -> CPDResult:
                                                 list(map(lambda x: x.get_task_name(), tasks['metrics'])))
         for i in range(0, len(exception_list)):
             self._logger.info(f"Error {i}")
-            self._logger.exception(exception_list[i])
+            self._logger.exception(exception_list[i], exc_info=exception_list[i])
         return validation_result

src/cpdbench/dataset/CPD2DFromFileDataset.py

@@ -9,7 +9,7 @@ class CPD2DFromFileDataset(CPDDataset):
     into the main memory. Instead numpy will lazy load all needed data points.
     """
 
-    def __init__(self, file_path: str, dtype: str, ground_truths: list[int]):
+    def __init__(self, file_path: str, dtype: str, ground_truths: list[int], validation_amount=-1):
         """Constructor
         :param file_path: The absolute or relative path to numpy file.
         :param dtype: The data type in which the numpy array was saved.
@@ -19,12 +19,20 @@ def __init__(self, file_path: str, dtype: str, ground_truths: list[int]):
         self.dtype = dtype
         self._array = None
         self._ground_truths = ground_truths
+        self._validation_amount = validation_amount
+
 
     def init(self) -> None:
         self._array = memmap(self.file_path, self.dtype, mode='r')
+        if self._validation_amount == -1:
+            self._validation_array = self._array[:]
+        else:
+            self._validation_array = self._array[0:self._validation_amount]
+        validation_array_length = self._validation_array.shape[0]
+        self._validation_ground_truths = [el for el in self._ground_truths if el < validation_array_length]
 
     def get_signal(self) -> tuple[ndarray, list[int]]:
         return self._array, self._ground_truths
 
     def get_validation_preview(self) -> tuple[ndarray, list[int]]:
-        return self._array, self._ground_truths
+        return self._validation_array, self._validation_ground_truths

src/cpdbench/dataset/CPDNdarrayDataset.py

@@ -0,0 +1,25 @@
+from numpy import ndarray
+
+from cpdbench.dataset.CPDDataset import CPDDataset
+
+
+class CPDNdarrayDataset(CPDDataset):
+
+    def get_validation_preview(self) -> tuple[ndarray, list[int]]:
+        return self._validation_array, self._validation_ground_truths
+
+    def __init__(self, numpy_array, ground_truths, validation_amount=-1):
+        self._ndarray = numpy_array
+        self._ground_truths = ground_truths
+        if validation_amount == -1:
+            self._validation_array = self._ndarray[:, :]
+        else:
+            self._validation_array = self._ndarray[:, 0:validation_amount]
+        validation_array_length = self._validation_array.shape[1]
+        self._validation_ground_truths = [el for el in self._ground_truths if el < validation_array_length]
+
+    def init(self) -> None:
+        pass
+
+    def get_signal(self) -> tuple[ndarray, list[int]]:
+        return self._ndarray, self._ground_truths

src/cpdbench/examples/ExampleAlgorithms.py

@@ -9,17 +9,7 @@ def numpy_array_accesses(dataset, array_indexes):
     return indexes, confidences
 
 
-def algorithm_execute_single_esst(signal):
-    """Uses SST as implemented in the changepoynt library as algorithm."""
-    detector = SST(90, method='rsvd')
-    sig = signal[0]
-    res = detector.transform(sig)
-    indexes = [res.argmax()]
-    confidences = [1.0]
-    return indexes, confidences
-
-
-def algorithm_execute_single_esst(signal, window_length):
+def algorithm_execute_single_esst(signal, window_length=90):
     """Uses SST as implemented in the changepoynt library as algorithm."""
     detector = SST(window_length, method='rsvd')
     sig = signal[0]

src/cpdbench/examples/ExampleDatasets.py

@@ -3,24 +3,24 @@
 import numpy as np
 
 from cpdbench.dataset.CPD2DFromFileDataset import CPD2DFromFileDataset
-from cpdbench.dataset.CPD2DNdarrayDataset import CPD2DNdarrayDataset
+from cpdbench.dataset.CPDNdarrayDataset import CPDNdarrayDataset
 
 
-def get_extreme_large_dataset_from_file():
+def get_extreme_large_dataset_from_file(validation_amount=-1):
     path = pathlib.Path(__file__).parent.resolve()
     path = path.joinpath("data", "very_big_numpy_file.dat")
-    dataset = CPD2DFromFileDataset(str(path), "float32", [5, 245, 255, 256, 25])
+    dataset = CPD2DFromFileDataset(str(path), "float32", [5, 245, 255, 256, 25], validation_amount)
     return dataset
 
 def dataset_get_apple_dataset():
     raw_data = np.load("../../../data/apple.npy")
     timeseries = raw_data[:, 0]
     reshaped_ts = np.reshape(timeseries, [1, timeseries.size])
-    return CPD2DNdarrayDataset(reshaped_ts, [337])
+    return CPDNdarrayDataset(reshaped_ts, [337])
 
 
 def dataset_get_bitcoin_dataset():
     raw_data = np.load("../../../data/bitcoin.npy")
     timeseries = raw_data[:, 0]
     reshaped_ts = np.reshape(timeseries, [1, timeseries.size])
-    return CPD2DNdarrayDataset(reshaped_ts, [569])
+    return CPDNdarrayDataset(reshaped_ts, [569])

src/cpdbench/examples/Example_Exception.py

@@ -0,0 +1,31 @@
+from cpdbench.CPDBench import CPDBench
+import cpdbench.examples.ExampleDatasets as example_datasets
+import cpdbench.examples.ExampleAlgorithms as example_algorithms
+import cpdbench.examples.ExampleMetrics as example_metrics
+
+cpdb = CPDBench()
+
+
+@cpdb.dataset
+def get_apple_dataset():
+    return example_datasets.dataset_get_apple_dataset()
+
+
+@cpdb.dataset
+def get_bitcoin_dataset():
+    raise KeyError
+    return example_datasets.dataset_get_bitcoin_dataset()
+
+
+@cpdb.algorithm
+def execute_esst_test(signal):
+    return example_algorithms.algorithm_execute_single_esst(signal)
+
+
+@cpdb.metric
+def calc_accuracy(indexes, scores, ground_truth):
+    return example_metrics.metric_accuracy_in_allowed_windows(indexes, scores, ground_truth, window_size=25)
+
+
+if __name__ == '__main__':
+    cpdb.start()

src/cpdbench/examples/Example_Parallelism.py

@@ -0,0 +1,33 @@
+from time import sleep
+
+from cpdbench.CPDBench import CPDBench
+import cpdbench.examples.ExampleDatasets as example_datasets
+import cpdbench.examples.ExampleAlgorithms as example_algorithms
+import cpdbench.examples.ExampleMetrics as example_metrics
+
+cpdb = CPDBench()
+
+
+@cpdb.dataset
+def get_apple_dataset():
+    sleep(10)
+    return example_datasets.dataset_get_apple_dataset()
+
+
+@cpdb.dataset
+def get_bitcoin_dataset():
+    return example_datasets.dataset_get_bitcoin_dataset()
+
+
+@cpdb.algorithm
+def execute_esst_test(signal):
+    return example_algorithms.algorithm_execute_single_esst(signal)
+
+
+@cpdb.metric
+def calc_accuracy(indexes, scores, ground_truth):
+    return example_metrics.metric_accuracy_in_allowed_windows(indexes, scores, ground_truth, window_size=25)
+
+
+if __name__ == '__main__':
+    cpdb.start()

src/cpdbench/examples/Example_Validation.py

@@ -0,0 +1,34 @@
+from cpdbench.CPDBench import CPDBench
+import cpdbench.examples.ExampleDatasets as example_datasets
+import cpdbench.examples.ExampleAlgorithms as example_algorithms
+import cpdbench.examples.ExampleMetrics as example_metrics
+
+cpdb = CPDBench()
+
+
+@cpdb.dataset
+def get_apple_dataset():
+    return example_datasets.dataset_get_apple_dataset()
+
+
+@cpdb.dataset
+def get_bitcoin_dataset():
+    return example_datasets.dataset_get_bitcoin_dataset()
+
+
+@cpdb.algorithm
+def execute_esst_test_wrong(signal, window):
+    return example_algorithms.algorithm_execute_single_esst(signal)
+
+@cpdb.algorithm
+def execute_esst_test(signal):
+    return example_algorithms.algorithm_execute_single_esst(signal)
+
+
+@cpdb.metric
+def calc_accuracy(indexes, scores, ground_truth):
+    return example_metrics.metric_accuracy_in_allowed_windows(indexes, scores, ground_truth, window_size=25)
+
+
+if __name__ == '__main__':
+    cpdb.start()

src/cpdbench/examples/Example_ValidationRuntime.py

@@ -0,0 +1,32 @@
+from cpdbench.examples import ExampleAlgorithms
+from cpdbench.examples.ExampleDatasets import get_extreme_large_dataset_from_file
+from cpdbench.examples.ExampleMetrics import metric_accuracy_in_allowed_windows
+from cpdbench.CPDBench import CPDBench
+import pathlib
+
+cpdb = CPDBench()
+
+
+@cpdb.dataset
+def get_large_dataset():
+    return get_extreme_large_dataset_from_file(1000)
+
+
+@cpdb.algorithm
+def execute_algorithm(dataset):
+    dataset = dataset.reshape((1, dataset.size))
+    res = ExampleAlgorithms.algorithm_execute_single_esst(dataset)
+    assert dataset.ndim == 3
+    return res
+
+
+@cpdb.metric
+def compute_metric(indexes, confidences, ground_truths):
+    return metric_accuracy_in_allowed_windows(indexes, confidences, ground_truths, window_size=20)
+
+
+if __name__ == '__main__':
+    path = pathlib.Path(__file__).parent.resolve()
+    path = path.joinpath("configs", "VeryLargeDatasetConfig.yml")
+    #cpdb.start(config_file=str(path))
+    cpdb.validate(config_file=str(path))

src/cpdbench/examples/configs/parametersConfig.yml

@@ -7,7 +7,6 @@ multiprocessing: True
 result:
   filename: cpdbench-result-parameters.json
 
-
 user:
   algorithm-executions:
     - window_length: 90

src/cpdbench/task/DatasetFetchTask.py

@@ -26,7 +26,7 @@ def validate_input(self, *args) -> CPDDataset:
             dataset.init()
         except Exception as e:
             raise DatasetValidationException(f"The validation of {get_name_of_function(self._function)} failed.") \
-                from e # TODO: Funktioniert das noch?
+                from e
         else:
             return dataset
 

src/cpdbench/task/TaskFactory.py

@@ -71,7 +71,7 @@ def create_tasks_with_parameters(self, function: Callable, task_type: TaskType)
             else:
                 for i in range(len(param_values)):
                     if param in global_params:
-                        param_values[i].update({param: vals[0]})  # global param # TODO: was wenn param wo fehlt?
+                        param_values[i].update({param: vals[0]})  # global param
                     else:
                         param_values[i].update({param: vals[i]})  # execution param