From f005422ba67f8e00689ce4de1fcd1c3aa7b5b84b Mon Sep 17 00:00:00 2001 From: cliveseldon Date: Tue, 14 Apr 2020 13:33:20 +0100 Subject: [PATCH] remove old outlier exmaples --- components/outlier-detection/README.md | 33 - .../isolation-forest/.s2i/environment | 4 - .../isolation-forest/CoreIsolationForest.py | 117 ---- .../OutlierIsolationForest.py | 115 ---- .../isolation-forest/README.md | 22 - .../isolation-forest/__init__.py | 0 .../outlier-detection/isolation-forest/doc.md | 130 ---- .../isolation-forest/isolation_forest.ipynb | 608 ----------------- .../isolation-forest/models/.keep | 0 .../isolation-forest/requirements.txt | 6 - .../isolation-forest/train.py | 77 --- .../isolation-forest/utils.py | 179 ----- .../mahalanobis/.s2i/environment | 4 - .../mahalanobis/CoreMahalanobis.py | 192 ------ .../mahalanobis/OutlierMahalanobis.py | 120 ---- .../outlier-detection/mahalanobis/README.md | 22 - .../outlier-detection/mahalanobis/__init__.py | 0 .../outlier-detection/mahalanobis/doc.ipynb | 350 ---------- .../images/outliers_3stdev_clipped.png | Bin 15130 -> 0 bytes .../images/outliers_no_clipping.png | Bin 18498 -> 0 bytes .../mahalanobis/outlier_mahalanobis.ipynb | 577 ---------------- .../mahalanobis/requirements.txt | 5 - .../outlier-detection/mahalanobis/utils.py | 171 ----- .../seq2seq-lstm/.s2i/environment | 4 - .../seq2seq-lstm/CoreSeq2SeqLSTM.py | 215 ------ .../seq2seq-lstm/OutlierSeq2SeqLSTM.py | 117 ---- .../outlier-detection/seq2seq-lstm/README.md | 24 - .../seq2seq-lstm/__init__.py | 0 .../outlier-detection/seq2seq-lstm/data/.keep | 0 .../outlier-detection/seq2seq-lstm/doc.md | 336 ---------- .../seq2seq-lstm/images/ecg.png | Bin 35448 -> 0 bytes .../seq2seq-lstm/images/inlier_ecg.png | Bin 19694 -> 0 bytes .../seq2seq-lstm/images/outlier_ecg.png | Bin 21501 -> 0 bytes .../outlier-detection/seq2seq-lstm/model.py | 93 --- .../seq2seq-lstm/models/.keep | 0 .../models/preprocess_seq2seq.pickle | Bin 245 -> 0 bytes .../seq2seq-lstm/models/seq2seq.pickle | Bin 38 -> 0 bytes .../seq2seq-lstm/models/seq2seq_weights.h5 | Bin 63704 -> 0 bytes .../seq2seq-lstm/requirements.txt | 8 - .../seq2seq-lstm/seq2seq_lstm.ipynb | 610 ----------------- .../outlier-detection/seq2seq-lstm/train.py | 155 ----- .../outlier-detection/seq2seq-lstm/utils.py | 91 --- .../outlier-detection/vae/.s2i/environment | 4 - components/outlier-detection/vae/CoreVAE.py | 182 ----- .../outlier-detection/vae/OutlierVAE.py | 119 ---- components/outlier-detection/vae/README.md | 22 - components/outlier-detection/vae/__init__.py | 0 components/outlier-detection/vae/doc.md | 292 -------- components/outlier-detection/vae/model.py | 92 --- .../vae/models/preprocess_vae.pickle | Bin 518 -> 0 bytes .../outlier-detection/vae/models/vae.pickle | Bin 34 -> 0 bytes .../vae/models/vae_weights.h5 | Bin 18864 -> 0 bytes .../outlier-detection/vae/outlier_vae.ipynb | 622 ------------------ .../outlier-detection/vae/requirements.txt | 7 - components/outlier-detection/vae/train.py | 147 ----- components/outlier-detection/vae/utils.py | 171 ----- 56 files changed, 6043 deletions(-) delete mode 100644 components/outlier-detection/README.md delete mode 100644 components/outlier-detection/isolation-forest/.s2i/environment delete mode 100644 components/outlier-detection/isolation-forest/CoreIsolationForest.py delete mode 100644 components/outlier-detection/isolation-forest/OutlierIsolationForest.py delete mode 100644 components/outlier-detection/isolation-forest/README.md delete mode 100644 components/outlier-detection/isolation-forest/__init__.py delete mode 100644 components/outlier-detection/isolation-forest/doc.md delete mode 100644 components/outlier-detection/isolation-forest/isolation_forest.ipynb delete mode 100644 components/outlier-detection/isolation-forest/models/.keep delete mode 100644 components/outlier-detection/isolation-forest/requirements.txt delete mode 100644 components/outlier-detection/isolation-forest/train.py delete mode 100644 components/outlier-detection/isolation-forest/utils.py delete mode 100644 components/outlier-detection/mahalanobis/.s2i/environment delete mode 100644 components/outlier-detection/mahalanobis/CoreMahalanobis.py delete mode 100644 components/outlier-detection/mahalanobis/OutlierMahalanobis.py delete mode 100644 components/outlier-detection/mahalanobis/README.md delete mode 100644 components/outlier-detection/mahalanobis/__init__.py delete mode 100644 components/outlier-detection/mahalanobis/doc.ipynb delete mode 100644 components/outlier-detection/mahalanobis/images/outliers_3stdev_clipped.png delete mode 100644 components/outlier-detection/mahalanobis/images/outliers_no_clipping.png delete mode 100644 components/outlier-detection/mahalanobis/outlier_mahalanobis.ipynb delete mode 100644 components/outlier-detection/mahalanobis/requirements.txt delete mode 100644 components/outlier-detection/mahalanobis/utils.py delete mode 100644 components/outlier-detection/seq2seq-lstm/.s2i/environment delete mode 100644 components/outlier-detection/seq2seq-lstm/CoreSeq2SeqLSTM.py delete mode 100644 components/outlier-detection/seq2seq-lstm/OutlierSeq2SeqLSTM.py delete mode 100644 components/outlier-detection/seq2seq-lstm/README.md delete mode 100644 components/outlier-detection/seq2seq-lstm/__init__.py delete mode 100644 components/outlier-detection/seq2seq-lstm/data/.keep delete mode 100644 components/outlier-detection/seq2seq-lstm/doc.md delete mode 100644 components/outlier-detection/seq2seq-lstm/images/ecg.png delete mode 100644 components/outlier-detection/seq2seq-lstm/images/inlier_ecg.png delete mode 100644 components/outlier-detection/seq2seq-lstm/images/outlier_ecg.png delete mode 100644 components/outlier-detection/seq2seq-lstm/model.py delete mode 100644 components/outlier-detection/seq2seq-lstm/models/.keep delete mode 100644 components/outlier-detection/seq2seq-lstm/models/preprocess_seq2seq.pickle delete mode 100644 components/outlier-detection/seq2seq-lstm/models/seq2seq.pickle delete mode 100644 components/outlier-detection/seq2seq-lstm/models/seq2seq_weights.h5 delete mode 100644 components/outlier-detection/seq2seq-lstm/requirements.txt delete mode 100644 components/outlier-detection/seq2seq-lstm/seq2seq_lstm.ipynb delete mode 100644 components/outlier-detection/seq2seq-lstm/train.py delete mode 100644 components/outlier-detection/seq2seq-lstm/utils.py delete mode 100644 components/outlier-detection/vae/.s2i/environment delete mode 100644 components/outlier-detection/vae/CoreVAE.py delete mode 100644 components/outlier-detection/vae/OutlierVAE.py delete mode 100644 components/outlier-detection/vae/README.md delete mode 100644 components/outlier-detection/vae/__init__.py delete mode 100644 components/outlier-detection/vae/doc.md delete mode 100644 components/outlier-detection/vae/model.py delete mode 100644 components/outlier-detection/vae/models/preprocess_vae.pickle delete mode 100644 components/outlier-detection/vae/models/vae.pickle delete mode 100644 components/outlier-detection/vae/models/vae_weights.h5 delete mode 100644 components/outlier-detection/vae/outlier_vae.ipynb delete mode 100644 components/outlier-detection/vae/requirements.txt delete mode 100644 components/outlier-detection/vae/train.py delete mode 100644 components/outlier-detection/vae/utils.py diff --git a/components/outlier-detection/README.md b/components/outlier-detection/README.md deleted file mode 100644 index 987d98a553..0000000000 --- a/components/outlier-detection/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Outlier Detection in Seldon Core - -## Description - -[Anomaly or outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. Seldon Core provides a number of outlier detectors suitable for different use cases. The detectors can be run as models or transformers which are part of the pre-defined types of [predictive units](../../docs/reference/seldon-deployment.md#proto-buffer-definition) in Seldon Core. Models are microservices that make predictions and can receive feedback rewards while the input transformers add the anomaly predictions to the metadata of the underlying model. The REST and gRPC internal APIs that the model and transformer components must conform to are covered in the [internal API](../../docs/reference/internal-api.md) reference. - -## Implementations - -The following types of outlier detectors are implemented and showcased with demos on Seldon Core: -* [Sequence-to-Sequence LSTM](./seq2seq-lstm) -* [Variational Auto-Encoder](./vae) -* [Isolation Forest](./isolation-forest) -* [Mahalanobis Distance](./mahalanobis) - -The Sequence-to-Sequence LSTM algorithm can be used to detect outliers in time series data, while the other algorithms spot anomalies in tabular data. The Mahalanobis detector works online and does not need to be trained first. The other algorithms are ideally trained on a batch of normal data or data with a low fraction of outliers. - -## Implementing custom outlier detectors - -An outlier detection component can be implemented either as a model or input transformer component. If the component is defined as a model, a ```predict``` method needs to be implemented to return the detected anomalies. Optionally, a ```send_feedback``` method can return additional information about the performance of the algorithm. When the component is used as a transformer, the anomaly predictions will occur in the ```transform_input``` method which returns the unchanged input features. The anomaly predictions will then be added to the underlying model's metadata via the ```tags``` method. Both models and transformers can make use of custom metrics defined by the ```metrics``` function. - -The required methods to use the outlier detection algorithms as models or transformers are implemented in the Python files with the ```Core``` prefix. The demos contain clear instructions on how to run your component as a model or transformer. - -## Language specific templates - -Reference templates for custom model and input transformer components written in several languages are available: -* Python - * [model](../../wrappers/s2i/python/test/model-template-app/MyModel.py) - * [transformer](../../wrappers/s2i/python/test/transformer-template-app/MyTransformer.py) -* R - * [model](../../wrappers/s2i/R/test/model-template-app/MyModel.R) - * [transformer](../../wrappers/s2i/R/test/transformer-template-app/MyTransformer.R) - -Additionally, the [wrappers](../../wrappers/s2i) provide guidelines for implementing the model component in other languages. \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/.s2i/environment b/components/outlier-detection/isolation-forest/.s2i/environment deleted file mode 100644 index ce589f54a3..0000000000 --- a/components/outlier-detection/isolation-forest/.s2i/environment +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=OutlierIsolationForest -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/components/outlier-detection/isolation-forest/CoreIsolationForest.py b/components/outlier-detection/isolation-forest/CoreIsolationForest.py deleted file mode 100644 index 0db0fca41c..0000000000 --- a/components/outlier-detection/isolation-forest/CoreIsolationForest.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -import numpy as np -import pickle -from sklearn.ensemble import IsolationForest - -logger = logging.getLogger(__name__) - -class CoreIsolationForest(object): - """ Outlier detection using Isolation Forests. - - Parameters - ---------- - threshold (float) : anomaly score threshold; scores below threshold are outliers - - Functions - ---------- - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=0.,model_name='if',load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.N = 0 # total sample count up until now - self.nb_outliers = 0 - - # load pre-trained model - with open(load_path + model_name + '.pickle', 'rb') as f: - self.clf = pickle.load(f) - - - def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) - - - def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X - - - def _get_preds(self,X): - """ Detect outliers below the anomaly score threshold. - - Parameters - ---------- - X : array-like - """ - self.decision_val = self.clf.decision_function(X) # anomaly scores - - # make prediction - self.prediction = (self.decision_val < self.threshold).astype(int) # scores below threshold are outliers - - self.N+=self.prediction.shape[0] # update counter - - return self.prediction - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return additional data as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - logger.info("Send feedback called") - return [] - - - def tags(self): - """ - Use predictions made within transform to add these as metadata - to the response. Tags will only be collected if the component is - used as an input-transformer. - """ - try: - return {"outlier-predictions": self.prediction_meta.tolist()} - except AttributeError: - logger.info("No metadata about outliers") - - - def metrics(self): - """ Return custom metrics averaged over the prediction batch. - """ - self.nb_outliers += np.sum(self.prediction) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":np.mean(self.prediction)} - anomaly_score = {"type":"GAUGE","key":"anomaly_score","value":np.mean(self.decision_val)} - nb_outliers = {"type":"GAUGE","key":"nb_outliers","value":int(self.nb_outliers)} - fraction_outliers = {"type":"GAUGE","key":"fraction_outliers","value":int(self.nb_outliers)/self.N} - obs = {"type":"GAUGE","key":"observation","value":self.N} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - return [is_outlier,anomaly_score,nb_outliers,fraction_outliers,obs,threshold] \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/OutlierIsolationForest.py b/components/outlier-detection/isolation-forest/OutlierIsolationForest.py deleted file mode 100644 index a56ba32085..0000000000 --- a/components/outlier-detection/isolation-forest/OutlierIsolationForest.py +++ /dev/null @@ -1,115 +0,0 @@ -import numpy as np - -from CoreIsolationForest import CoreIsolationForest -from utils import flatten, performance, outlier_stats - -class OutlierIsolationForest(CoreIsolationForest): - """ Outlier detection using Isolation Forests. - - Parameters - ---------- - threshold (float) : anomaly score threshold; scores below threshold are outliers - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - def __init__(self,threshold=0.,model_name='if',load_path='./models/'): - - super().__init__(threshold=threshold, model_name=model_name, load_path=load_path) - - self._predictions = [] - self._labels = [] - self._anomaly_score = [] - self.roll_window = 100 - self.metric = [float('nan') for i in range(18)] - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return outlier labels as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - _ = super().send_feedback(X,feature_names,reward,truth) - - # historical reconstruction errors and predictions - self._anomaly_score.append(self.decision_val) - self._anomaly_score = flatten(self._anomaly_score) - self._predictions.append(self.prediction) - self._predictions = flatten(self._predictions) - - # target labels - self.label = truth - self._labels.append(self.label) - self._labels = flatten(self._labels) - - # performance metrics - scores = performance(self._labels,self._predictions,roll_window=self.roll_window) - stats = outlier_stats(self._labels,self._predictions,roll_window=self.roll_window) - - convert = flatten([scores,stats]) - metric = [] - for c in convert: # convert from np to native python type to jsonify - metric.append(np.asscalar(np.asarray(c))) - self.metric = metric - - return [] - - - def metrics(self): - """ Return custom metrics. - Printed with a delay of 1 prediction because the labels are returned in the feedback step. - """ - - if self.prediction.shape[0]>1: - raise ValueError('Metrics can only handle single observations.') - - if self.N==1: - pred = float('nan') - dec_val = float('nan') - y_true = float('nan') - else: - pred = int(self._predictions[-1]) - dec_val = self._anomaly_score[-1] - y_true = int(self.label[0]) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":pred} - anomaly_score = {"type":"GAUGE","key":"anomaly_score","value":dec_val} - obs = {"type":"GAUGE","key":"observation","value":self.N - 1} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - label = {"type":"GAUGE","key":"label","value":y_true} - - accuracy_tot = {"type":"GAUGE","key":"accuracy_tot","value":self.metric[4]} - precision_tot = {"type":"GAUGE","key":"precision_tot","value":self.metric[5]} - recall_tot = {"type":"GAUGE","key":"recall_tot","value":self.metric[6]} - f1_score_tot = {"type":"GAUGE","key":"f1_tot","value":self.metric[7]} - f2_score_tot = {"type":"GAUGE","key":"f2_tot","value":self.metric[8]} - - accuracy_roll = {"type":"GAUGE","key":"accuracy_roll","value":self.metric[9]} - precision_roll = {"type":"GAUGE","key":"precision_roll","value":self.metric[10]} - recall_roll = {"type":"GAUGE","key":"recall_roll","value":self.metric[11]} - f1_score_roll = {"type":"GAUGE","key":"f1_roll","value":self.metric[12]} - f2_score_roll = {"type":"GAUGE","key":"f2_roll","value":self.metric[13]} - - true_negative = {"type":"GAUGE","key":"true_negative","value":self.metric[0]} - false_positive = {"type":"GAUGE","key":"false_positive","value":self.metric[1]} - false_negative = {"type":"GAUGE","key":"false_negative","value":self.metric[2]} - true_positive = {"type":"GAUGE","key":"true_positive","value":self.metric[3]} - - nb_outliers_roll = {"type":"GAUGE","key":"nb_outliers_roll","value":self.metric[14]} - nb_labels_roll = {"type":"GAUGE","key":"nb_labels_roll","value":self.metric[15]} - nb_outliers_tot = {"type":"GAUGE","key":"nb_outliers_tot","value":self.metric[16]} - nb_labels_tot = {"type":"GAUGE","key":"nb_labels_tot","value":self.metric[17]} - - return [is_outlier,anomaly_score,obs,threshold,label, - accuracy_tot,precision_tot,recall_tot,f1_score_tot,f2_score_tot, - accuracy_roll,precision_roll,recall_roll,f1_score_roll,f2_score_roll, - true_negative,false_positive,false_negative,true_positive, - nb_outliers_roll,nb_labels_roll,nb_outliers_tot,nb_labels_tot] \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/README.md b/components/outlier-detection/isolation-forest/README.md deleted file mode 100644 index 0ec8c10f10..0000000000 --- a/components/outlier-detection/isolation-forest/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Isolation Forest Outlier Detector - -## Description - -[Anomaly or outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The implemented [Isolation Forest](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html) outlier detector aims to predict anomalies in tabular data. The anomaly detector predicts whether the input features represent normal behaviour or not, dependent on a threshold level set by the user. - -## Implementation - -The Isolation Forest is trained by running the ```train.py``` script. The ```OutlierIsolationForest``` class inherits from ```CoreIsolationForest``` which loads a pre-trained model and can make predictions on new data. - -A detailed explanation of the implementation and usage of Isolation Forests as outlier detectors can be found in the [isolation forest doc](./doc.md). - -## Running on Seldon - -An end-to-end example running an Isolation Forest outlier detector on GCP or Minikube using Seldon to identify computer network intrusions is available [here](./isolation_forest.ipynb). - -Docker images to use the generic Isolation Forest outlier detector as a model or transformer can be found on Docker Hub: -* [seldonio/outlier-if-model](https://hub.docker.com/r/seldonio/outlier-if-model) -* [seldonio/outlier-if-transformer](https://hub.docker.com/r/seldonio/outlier-if-transformer) - -A model docker image specific for the demo is also available: -* [seldonio/outlier-if-model-demo](https://hub.docker.com/r/seldonio/outlier-if-model-demo) \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/__init__.py b/components/outlier-detection/isolation-forest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/isolation-forest/doc.md b/components/outlier-detection/isolation-forest/doc.md deleted file mode 100644 index 111341ed7f..0000000000 --- a/components/outlier-detection/isolation-forest/doc.md +++ /dev/null @@ -1,130 +0,0 @@ -# Isolation Forest (IF) Algorithm Documentation - -The aim of this document is to explain the Isolation Forest algorithm in Seldon's outlier detection framework. - -First, we provide a high level overview of the algorithm and the use case, then we will give a detailed explanation of the implementation. - -## Overview - -Outlier detection has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The available data is typically unlabeled and detection needs to be done in real-time. The outlier detector can be used as a standalone algorithm, or to detect anomalies in the input data of another predictive model. - -The IF outlier detection algorithm predicts whether the input features are an outlier or not, dependent on a threshold level set by the user. The algorithm needs to be pretrained first on a representable batch of data. - -As observations arrive, the algorithm will: -- calculate an anomaly score for the observation -- predict that the observation is an outlier if the anomaly score is below the threshold level - -## Why Isolation Forests? - -Isolation forests are tree based models specifically used for outlier detection. The IF isolates observations by randomly selecting a feature and then randomly selecting a split value between the maximum and minimum values of the selected feature. The number of splittings required to isolate a sample is equivalent to the path length from the root node to the terminating node. This path length, averaged over a forest of random trees, is a measure of normality and is used to define an anomaly score. Outliers can typically be isolated quicker, leading to shorter paths. In the scikit-learn implementation, lower anomaly scores indicate that the probability of an observation being an outlier is higher. - -## Implementation - -### 1. Defining and training the IF model - -The model takes 4 hyperparameters: - -- contamination: the fraction of expected outliers in the data set -- number of estimators: the number of base estimators; number of trees in the forest -- max samples: fraction of samples used for each base estimator -- max features: fraction of features used for each base estimator - -``` python -!python train.py \ ---dataset 'kddcup99' \ ---samples 50000 \ ---keep_cols "$cols_str" \ ---contamination .1 \ ---n_estimators 100 \ ---max_samples .8 \ ---max_features 1. \ ---save_path './models/' -``` - -The model is saved in the folder specified by "save_path". - -### 2. Making predictions - -In order to make predictions, which can then be served by Seldon Core, the pre-trained model is loaded when defining an OutlierIsolationForest object. The "threshold" argument defines below which anomaly score a sample is classified as an outlier. The threshold is a key hyperparameter and needs to be picked carefully for each application. The OutlierIsolationForest class inherits from the CoreIsolationForest class in ```CoreIsolationForest.py```. - -``` python -class CoreIsolationForest(object): - """ Outlier detection using Isolation Forests. - - Parameters - ---------- - threshold (float) : anomaly score threshold; scores below threshold are outliers - - Functions - ---------- - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=0.,load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.N = 0 # total sample count up until now - self.nb_outliers = 0 - - # load pre-trained model - with open(load_path + 'model.pickle', 'rb') as f: - self.clf = pickle.load(f) -``` - -```python -class OutlierIsolationForest(CoreIsolationForest): - """ Outlier detection using Isolation Forests. - - Parameters - ---------- - threshold (float) : anomaly score threshold; scores below threshold are outliers - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - def __init__(self,threshold=0.,load_path='./models/'): - - super().__init__(threshold=threshold, load_path=load_path) -``` - -The actual outlier detection is done by the ```_get_preds``` method which is invoked by ```predict``` or ```transform_input``` dependent on whether the detector is defined as respectively a model or a transformer. - -``` python -def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) -``` - -```python -def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X -``` - -## References - -Scikit-learn Isolation Forest: -- https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/isolation_forest.ipynb b/components/outlier-detection/isolation-forest/isolation_forest.ipynb deleted file mode 100644 index d0aecb7606..0000000000 --- a/components/outlier-detection/isolation-forest/isolation_forest.ipynb +++ /dev/null @@ -1,608 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Isolation Forest (IF) outlier detector deployment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap a scikit-learn Isolation Forest python model for use as a prediction microservice in seldon-core and deploy on seldon-core running on minikube or a Kubernetes cluster using GCP." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- [helm](https://github.com/helm/helm)\n", - "- [minikube](https://github.com/kubernetes/minikube) \n", - "- [s2i](https://github.com/openshift/source-to-image) >= 1.1.13 \n", - "\n", - "python packages:\n", - "- scikit-learn: pip install scikit-learn --> 0.20.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Task" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier detector needs to detect computer network intrusions using TCP dump data for a local-area network (LAN) simulating a typical U.S. Air Force LAN. A connection is a sequence of TCP packets starting and ending at some well defined times, between which data flows to and from a source IP address to a target IP address under some well defined protocol. Each connection is labeled as either normal, or as an attack. \n", - "\n", - "There are 4 types of attacks in the dataset:\n", - "- DOS: denial-of-service, e.g. syn flood;\n", - "- R2L: unauthorized access from a remote machine, e.g. guessing password;\n", - "- U2R: unauthorized access to local superuser (root) privileges;\n", - "- probing: surveillance and other probing, e.g., port scanning.\n", - " \n", - "The dataset contains about 5 million connection records.\n", - "\n", - "There are 3 types of features:\n", - "- basic features of individual connections, e.g. duration of connection\n", - "- content features within a connection, e.g. number of failed log in attempts\n", - "- traffic features within a 2 second window, e.g. number of connections to the same host as the current connection\n", - "\n", - "The outlier detector is only using 40 out of 41 features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train locally\n", - "\n", - "Train on small dataset where you roughly know the fraction of outliers, defined by the \"contamination\" parameter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# define columns to keep\n", - "cols=['duration','protocol_type','flag','src_bytes','dst_bytes','land',\n", - " 'wrong_fragment','urgent','hot','num_failed_logins','logged_in',\n", - " 'num_compromised','root_shell','su_attempted','num_root','num_file_creations',\n", - " 'num_shells','num_access_files','num_outbound_cmds','is_host_login',\n", - " 'is_guest_login','count','srv_count','serror_rate','srv_serror_rate',\n", - " 'rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate',\n", - " 'srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate',\n", - " 'dst_host_diff_srv_rate','dst_host_same_src_port_rate','dst_host_srv_diff_host_rate',\n", - " 'dst_host_serror_rate','dst_host_srv_serror_rate','dst_host_rerror_rate',\n", - " 'dst_host_srv_rerror_rate','target']\n", - "cols_str = str(cols)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python train.py \\\n", - "--dataset 'kddcup99' \\\n", - "--samples 50000 \\\n", - "--keep_cols \"$cols_str\" \\\n", - "--contamination .1 \\\n", - "--n_estimators 100 \\\n", - "--max_samples .8 \\\n", - "--max_features 1. \\\n", - "--save_path './models/'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test using Kubernetes cluster on GCP or Minikube" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run the outlier detector as a model or a transformer. If you want to run the anomaly detector as a transformer, change the SERVICE_TYPE variable from MODEL to TRANSFORMER [here](./.s2i/environment), set MODEL = False and change ```OutlierIsolationForest.py``` to:\n", - "\n", - "```python\n", - "from CoreIsolationForest import CoreIsolationForest\n", - "\n", - "class OutlierIsolationForest(CoreIsolationForest):\n", - " \"\"\" Outlier detection using Isolation Forests.\n", - "\n", - " Parameters\n", - " ----------\n", - " threshold (float) : anomaly score threshold; scores below threshold are outliers\n", - " \"\"\"\n", - " def __init__(self,threshold=0.,load_path='./models/'):\n", - "\n", - " super().__init__(threshold=threshold, load_path=load_path)\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pick Kubernetes cluster on GCP or Minikube." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MINIKUBE = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube start --memory 4096\n", - "else:\n", - " !gcloud container clusters get-credentials standard-cluster-1 --zone europe-west1-b --project seldon-demos" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a cluster-wide cluster-admin role assigned to a service account named “default” in the namespace “kube-system”." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin \\\n", - "--serviceaccount=kube-system:default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add current context details to the configuration file in the seldon namespace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create tiller service account and give it a cluster-wide cluster-admin role." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl -n kube-system create sa tiller\n", - "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n", - "!helm init --service-account tiller" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status and deploy seldon/spartakus helm charts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/tiller-deploy -n kube-system" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usage_metrics.enabled=true --namespace seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status for seldon core." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install Ambassador API gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!helm install stable/ambassador --name ambassador --set crds.keep=false" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deployment.apps/ambassador" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If Minikube used: create docker image for outlier detector inside Minikube using s2i. Besides the transformer image and the demo specific model image, the general model image for the Isolation Forest outlier detector is also available from Docker Hub as ***seldonio/outlier-if-model:0.1***." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if MINIKUBE & MODEL:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-if-model-demo:0.1\n", - "elif MINIKUBE:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-if-transformer:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install outlier detector helm charts either as a model or transformer and set *threshold* hyperparameter value." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " !helm install ../../../helm-charts/seldon-od-model \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set model.type=isolationforest \\\n", - " --set model.isolationforest.image.name=seldonio/outlier-if-model-demo:0.1 \\\n", - " --set model.isolationforest.threshold=0 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set replicas=1\n", - "else:\n", - " !helm install ../../../helm-charts/seldon-od-transformer \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set outlierDetection.enabled=true \\\n", - " --set outlierDetection.name=outlier-if \\\n", - " --set outlierDetection.type=isolationforest \\\n", - " --set outlierDetection.isolationforest.image.name=seldonio/outlier-if-transformer:0.1 \\\n", - " --set outlierDetection.isolationforest.threshold=0 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set model.image.name=seldonio/mock_classifier:1.0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Ambassador\n", - "\n", - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import rest requests, load data and test requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from utils import get_payload, rest_request_ambassador, send_feedback_rest, get_kdd_data, generate_batch\n", - "\n", - "data = get_kdd_data(keep_cols=cols,percent10=True) # load dataset\n", - "print(data.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate a random batch from the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "samples = 1\n", - "fraction_outlier = 0.\n", - "X, labels = generate_batch(data,samples,fraction_outlier)\n", - "print(X.shape)\n", - "print(labels.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test the rest requests with the generated data. It is important that the order of requests is respected. First we make predictions, then we get the \"true\" labels back using the feedback request. If we do not respect the order and eg keep making predictions without getting the feedback for each prediction, there will be a mismatch between the predicted and \"true\" labels. This will result in errors in the produced metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "request = get_payload(X)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the outlier detector is used as a transformer, the output of the anomaly detection is added as part of the metadata. If it is used as a model, we send model feedback to retrieve custom performance metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analytics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install the helm charts for prometheus and the grafana dashboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", - " --set grafana_prom_admin_password=password \\\n", - " --set persistence.enabled=false \\\n", - " --namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Grafana dashboard" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n", - "\n", - "Username : admin\n", - "\n", - "password : password (as set when starting seldon-core-analytics above)\n", - "\n", - "Import the outlier-detector-if dashboard from ../../../helm-charts/seldon-core-analytics/files/grafana/configs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run simulation\n", - "\n", - "- Sample random network intrusion data with a certain outlier probability.\n", - "- Get payload for the observation.\n", - "- Make a prediction.\n", - "- Send the \"true\" label with the feedback if the detector is run as a model.\n", - "\n", - "It is important that the prediction-feedback order is maintained. Otherwise there will be a mismatch between the predicted and \"true\" labels.\n", - "\n", - "View the progress on the grafana \"Outlier Detection\" dashboard. Most metrics need the outlier detector to be run as a model since they need model feedback." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "n_requests = 100\n", - "samples = 1\n", - "for i in range(n_requests):\n", - " fraction_outlier = .1\n", - " X, labels = generate_batch(data,samples,fraction_outlier)\n", - " request = get_payload(X)\n", - " response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")\n", - " if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube delete" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/components/outlier-detection/isolation-forest/models/.keep b/components/outlier-detection/isolation-forest/models/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/isolation-forest/requirements.txt b/components/outlier-detection/isolation-forest/requirements.txt deleted file mode 100644 index 772f39f27b..0000000000 --- a/components/outlier-detection/isolation-forest/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy==1.14.5 -argparse==1.1 -pandas==0.23.4 -scikit-learn==0.20.1 -scipy==1.1.0 -requests>=2.20.0 \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/train.py b/components/outlier-detection/isolation-forest/train.py deleted file mode 100644 index e799a37120..0000000000 --- a/components/outlier-detection/isolation-forest/train.py +++ /dev/null @@ -1,77 +0,0 @@ -import argparse -import numpy as np -import pickle -import random -from sklearn.ensemble import IsolationForest - -from utils import get_kdd_data, generate_batch - -np.random.seed(2018) -np.random.RandomState(2018) -random.seed(2018) - -# default args -DATASET = 'kddcup99' -SAMPLES = 50000 -COLS = str(['duration','protocol_type','flag','src_bytes','dst_bytes','land','wrong_fragment','urgent','hot', - 'num_failed_logins','logged_in','num_compromised','root_shell','su_attempted','num_root','num_file_creations', - 'num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','count','srv_count', - 'serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate', - 'srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate','dst_host_diff_srv_rate', - 'dst_host_same_src_port_rate','dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target']) -MODEL_NAME = 'if' -SAVE = True -SAVE_PATH = './models/' - -# Isolation Forest hyperparameters -CONTAMINATION = .1 -N_ESTIMATORS = 50 -MAX_SAMPLES = .8 -MAX_FEATURES = 1. - -def train(X,args): - """ Fit Isolation Forest. """ - - clf = IsolationForest(n_estimators=args.n_estimators, max_samples=args.max_samples, max_features=args.max_features, - contamination=args.contamination,behaviour='new') - clf.fit(X) - - if args.save: # save model - with open(args.save_path + args.model_name + '.pickle', 'wb') as f: - pickle.dump(clf,f) - -def run(args): - """ Load data, generate training batch and train Isolation Forest. """ - - print('\nLoad dataset') - if args.dataset=='kddcup99': - keep_cols = args.keep_cols[1:-1].replace("'","").replace(" ","").split(",") - data = get_kdd_data(keep_cols=keep_cols) - else: - raise ValueError('Only "kddcup99" dataset supported.') - - print('\nGenerate training batch') - X, _ = generate_batch(data,args.samples,args.contamination) - - print('\nTrain outlier detector') - train(X,args) - - print('\nTraining done!') - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description="Train Isolation Forest outlier detector.") - parser.add_argument('--dataset',type=str,choices=DATASET,default=DATASET) - parser.add_argument('--keep_cols',type=str,default=COLS) - parser.add_argument('--samples',type=int,default=SAMPLES) - parser.add_argument('--contamination',type=float,default=CONTAMINATION) - parser.add_argument('--n_estimators',type=int,default=N_ESTIMATORS) - parser.add_argument('--max_samples',type=float,default=MAX_SAMPLES) - parser.add_argument('--max_features',type=float,default=MAX_FEATURES) - parser.add_argument('--model_name',type=str,default=MODEL_NAME) - parser.add_argument('--save', default=SAVE, action='store_false') - parser.add_argument('--save_path',type=str,default=SAVE_PATH) - args = parser.parse_args() - - run(args) \ No newline at end of file diff --git a/components/outlier-detection/isolation-forest/utils.py b/components/outlier-detection/isolation-forest/utils.py deleted file mode 100644 index 4515e7ef3d..0000000000 --- a/components/outlier-detection/isolation-forest/utils.py +++ /dev/null @@ -1,179 +0,0 @@ -import collections -import json -import numpy as np -import pandas as pd -import requests -from sklearn.datasets import fetch_kddcup99 -from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score, fbeta_score - -pd.options.mode.chained_assignment = None # default='warn' - -def get_kdd_data(target=['dos','r2l','u2r','probe'], - keep_cols=['srv_count','serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate', - 'same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count', - 'dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate', - 'dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target'], - percent10=False): - """ Load KDD Cup 1999 data and return in dataframe. """ - - data_raw = fetch_kddcup99(subset=None, data_home=None, percent10=percent10) - - # specify columns - cols=['duration','protocol_type','service','flag','src_bytes','dst_bytes','land','wrong_fragment','urgent','hot', - 'num_failed_logins','logged_in','num_compromised','root_shell','su_attempted','num_root','num_file_creations', - 'num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','count','srv_count', - 'serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate', - 'srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate','dst_host_diff_srv_rate', - 'dst_host_same_src_port_rate','dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate'] - - # create dataframe - data = pd.DataFrame(data=data_raw['data'],columns=cols) - - # add target to dataframe - data['attack_type'] = data_raw['target'] - - # specify and map attack types - attack_list = np.unique(data['attack_type']) - attack_category = ['dos','u2r','r2l','r2l','r2l','probe','dos','u2r','r2l','dos','probe','normal','u2r', - 'r2l','dos','probe','u2r','probe','dos','r2l','dos','r2l','r2l'] - - attack_types = {} - for i,j in zip(attack_list,attack_category): - attack_types[i] = j - - data['attack_category'] = 'normal' - for key,value in attack_types.items(): - data['attack_category'][data['attack_type'] == key] = value - - # define target - data['target'] = 0 - for t in target: - data['target'][data['attack_category'] == t] = 1 - - # define columns to be dropped - drop_cols = [] - for col in data.columns.values: - if col not in keep_cols: - drop_cols.append(col) - - if drop_cols!=[]: - data.drop(columns=drop_cols,inplace=True) - - # apply OHE if necessary - cols_ohe = ['protocol_type','service','flag'] - for col in cols_ohe: - if col in keep_cols: - col_ohe = pd.get_dummies(data[col],prefix=col) - data = data.join(col_ohe) - data.drop([col],axis=1,inplace=True) - - return data - - -def sample_df(df,n): - """ Sample from df. """ - if n < df.shape[0]+1: - replace = False - else: - replace = True - return df.sample(n=n,replace=replace) - - -def generate_batch(data,n_samples,frac_outliers): - """ Generate random batch from data with fixed size and fraction of outliers. """ - - normal = data[data['target']==0] - outlier = data[data['target']==1] - - if n_samples==1: - n_outlier = np.random.binomial(1,frac_outliers) - n_normal = 1 - n_outlier - else: - n_normal = int((1-frac_outliers) * n_samples) - n_outlier = int(frac_outliers * n_samples) - - batch_normal = sample_df(normal,n_normal) - batch_outlier = sample_df(outlier,n_outlier) - - batch = pd.concat([batch_normal,batch_outlier]) - batch = batch.sample(frac=1).reset_index(drop=True) - - outlier_true = batch['target'].values - batch.drop(columns=['target'],inplace=True) - - return batch.values.astype('float'), outlier_true - -def flatten(x): - if isinstance(x, collections.Iterable): - return [a for i in x for a in flatten(i)] - else: - return [x] - -def performance(y_true,y_pred,roll_window=100): - """ Return a confusion matrix and calculate rolling accuracy, precision, recall, F1 and F2 scores. """ - - # confusion matrix - cm = confusion_matrix(y_true,y_pred,labels=[0,1]) - tn, fp, fn, tp = cm.ravel() - - # total scores - acc_tot = accuracy_score(y_true,y_pred) - prec_tot = precision_score(y_true,y_pred) - rec_tot = recall_score(y_true,y_pred) - f1_tot = f1_score(y_true,y_pred) - f2_tot = fbeta_score(y_true,y_pred,beta=2) - - # rolling scores - y_true_roll = y_true[-roll_window:] - y_pred_roll = y_pred[-roll_window:] - acc_roll = accuracy_score(y_true_roll,y_pred_roll) - prec_roll = precision_score(y_true_roll,y_pred_roll) - rec_roll = recall_score(y_true_roll,y_pred_roll) - f1_roll = f1_score(y_true_roll,y_pred_roll) - f2_roll = fbeta_score(y_true_roll,y_pred_roll,beta=2) - - scores = [tn, fp, fn, tp, acc_tot, prec_tot, rec_tot, f1_tot, f2_tot, - acc_roll, prec_roll, rec_roll, f1_roll, f2_roll] - - return scores - -def outlier_stats(y_true,y_pred,roll_window=100): - """ Calculate number and percentage of predicted and labeled outliers. """ - - y_pred_roll = np.sum(y_pred[-roll_window:]) - y_true_roll = np.sum(y_true[-roll_window:]) - y_pred_tot = np.sum(y_pred) - y_true_tot = np.sum(y_true) - - return y_pred_roll, y_true_roll, y_pred_tot, y_true_tot - -def get_payload(arr): - features = ["srv_count","serror_rate","srv_serror_rate","rerror_rate","srv_rerror_rate","same_srv_rate", - "diff_srv_rate","srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate", - "dst_host_diff_srv_rate","dst_host_same_src_port_rate","dst_host_srv_diff_host_rate", - "dst_host_serror_rate","dst_host_srv_serror_rate","dst_host_rerror_rate","dst_host_srv_rerror_rate"] - datadef = {"names":features,"ndarray":arr.tolist()} - payload = {"meta":{},"data":datadef} - return payload - -def rest_request_ambassador(deploymentName,namespace,request,endpoint="localhost:8003"): - response = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/predictions", - json=request) - print(response.status_code) - print(response.text) - return response.json() - -def send_feedback_rest(deploymentName,namespace,request,response,reward,truth,endpoint="localhost:8003"): - feedback = { - "request": request, - "response": response, - "reward": reward, - "truth": {"data":{"ndarray":truth.tolist()}} - } - ret = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/feedback", - json=feedback) - return diff --git a/components/outlier-detection/mahalanobis/.s2i/environment b/components/outlier-detection/mahalanobis/.s2i/environment deleted file mode 100644 index 8bbf65edae..0000000000 --- a/components/outlier-detection/mahalanobis/.s2i/environment +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=OutlierMahalanobis -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/components/outlier-detection/mahalanobis/CoreMahalanobis.py b/components/outlier-detection/mahalanobis/CoreMahalanobis.py deleted file mode 100644 index ac90c553de..0000000000 --- a/components/outlier-detection/mahalanobis/CoreMahalanobis.py +++ /dev/null @@ -1,192 +0,0 @@ -import logging -import numpy as np -from scipy.linalg import eigh - -logger = logging.getLogger(__name__) - -class CoreMahalanobis(object): - """ Outlier detection using the Mahalanobis distance. - - Parameters - ---------- - threshold (float) : Mahalanobis distance threshold used to classify outliers - n_components (int) : number of principal components used - n_stdev (float) : stdev used for feature-wise clipping of observations - start_clip (int) : number of observations before clipping is applied - max_n (int) : algorithm behaves as if it has seen at most max_n points - - Functions - ---------- - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - def __init__(self,threshold=25,n_components=3,n_stdev=3,start_clip=50,max_n=-1): - - logger.info("Initializing model") - self.threshold = threshold - self.n_components = n_components - self.max_n = max_n - self.n_stdev = n_stdev - self.start_clip = start_clip - - self.clip = None - self.mean = 0 - self.C = 0 - self.n = 0 - self.nb_outliers = 0 - - - def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) - - - def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X - - - def _get_preds(self,X): - """ Detect outliers using the Mahalanobis distance threshold. - - Parameters - ---------- - X : array-like - """ - - nb = X.shape[0] # batch size - p = X.shape[1] # number of features - n_components = min(self.n_components,p) - if self.max_n>0: - n = min(self.n,self.max_n) # n can never be above max_n - else: - n = self.n - - # Clip X - if self.n > self.start_clip: - Xclip = np.clip(X,self.clip[0],self.clip[1]) - else: - Xclip = X - - # Tracking the mean and covariance matrix - roll_partial_means = Xclip.cumsum(axis=0)/(np.arange(nb)+1).reshape((nb,1)) - coefs = (np.arange(nb)+1.)/(np.arange(nb)+n+1.) - new_means = self.mean + coefs.reshape((nb,1))*(roll_partial_means-self.mean) - new_means_offset = np.empty_like(new_means) - new_means_offset[0] = self.mean - new_means_offset[1:] = new_means[:-1] - - coefs = ((n+np.arange(nb))/(n+np.arange(nb)+1.)).reshape((nb,1,1)) - B = coefs*np.matmul((Xclip - new_means_offset)[:,:,None],(Xclip - new_means_offset)[:,None,:]) - cov_batch = (n-1.)/(n+max(1,nb-1.))*self.C + 1./(n+max(1,nb-1.))*B.sum(axis=0) - - # PCA - eigvals, eigvects = eigh(cov_batch,eigvals=(p-n_components,p-1)) - - # Projections - proj_x = np.matmul(X,eigvects) - proj_x_clip = np.matmul(Xclip,eigvects) - proj_means = np.matmul(new_means_offset,eigvects) - if type(self.C) == int and self.C == 0: - proj_cov = np.diag(np.zeros(n_components)) - else: - proj_cov = np.matmul(eigvects.transpose(),np.matmul(self.C,eigvects)) - - # Outlier detection in the PC subspace - coefs = (1./(n+np.arange(nb)+1.)).reshape((nb,1,1)) - B = coefs*np.matmul((proj_x_clip - proj_means)[:,:,None],(proj_x_clip - proj_means)[:,None,:]) - - all_C_inv = np.zeros_like(B) - c_inv = None - _EPSILON = 1e-8 - - for i, b in enumerate(B): - if c_inv is None: - if abs(np.linalg.det(proj_cov)) > _EPSILON: - c_inv = np.linalg.inv(proj_cov) - all_C_inv[i] = c_inv - continue - else: - if n + i == 0: - continue - proj_cov = (n + i -1. )/(n + i)*proj_cov + b - continue - else: - c_inv = (n + i - 1.)/float(n + i - 2.)*all_C_inv[i-1] - BC1 = np.matmul(B[i-1],c_inv) - all_C_inv[i] = c_inv - 1./(1.+np.trace(BC1))*np.matmul(c_inv,BC1) - - # Updates - self.mean = new_means[-1] - self.C = cov_batch - stdev = np.sqrt(np.diag(cov_batch)) - self.n += nb - if self.n > self.start_clip: - self.clip = [self.mean-self.n_stdev*stdev,self.mean+self.n_stdev*stdev] - - # Outlier scores and predictions - x_diff = proj_x-proj_means - self.score = np.matmul(x_diff[:,None,:],np.matmul(all_C_inv,x_diff[:,:,None])).reshape(nb) - self.prediction = np.array([1 if s > self.threshold else 0 for s in self.score]).astype(int) - - return self.prediction - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return additional data as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - logger.info("Send feedback called") - return [] - - - def tags(self): - """ - Use predictions made within transform to add these as metadata - to the response. Tags will only be collected if the component is - used as an input-transformer. - """ - try: - return {"outlier-predictions": self.prediction_meta.tolist()} - except AttributeError: - logger.info("No metadata about outliers") - - - def metrics(self): - """ Return custom metrics averaged over the prediction batch. - """ - self.nb_outliers += np.sum(self.prediction) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":np.mean(self.prediction)} - outlier_score = {"type":"GAUGE","key":"outlier_score","value":np.mean(self.score)} - nb_outliers = {"type":"GAUGE","key":"nb_outliers","value":int(self.nb_outliers)} - fraction_outliers = {"type":"GAUGE","key":"fraction_outliers","value":int(self.nb_outliers)/self.n} - obs = {"type":"GAUGE","key":"observation","value":self.n} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - return [is_outlier,outlier_score,nb_outliers,fraction_outliers,obs,threshold] \ No newline at end of file diff --git a/components/outlier-detection/mahalanobis/OutlierMahalanobis.py b/components/outlier-detection/mahalanobis/OutlierMahalanobis.py deleted file mode 100644 index 916440289e..0000000000 --- a/components/outlier-detection/mahalanobis/OutlierMahalanobis.py +++ /dev/null @@ -1,120 +0,0 @@ -import numpy as np - -from CoreMahalanobis import CoreMahalanobis -from utils import flatten, performance, outlier_stats - -class OutlierMahalanobis(CoreMahalanobis): - """ Outlier detection using the Mahalanobis distance. - - Parameters - ---------- - threshold (float) : Mahalanobis distance threshold used to classify outliers - n_components (int) : number of principal components used - n_stdev (float) : stdev used for feature-wise clipping of observations - start_clip (int) : number of observations before clipping is applied - max_n (int) : algorithm behaves as if it has seen at most max_n points - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - def __init__(self,threshold=25,n_components=3,n_stdev=3,start_clip=50,max_n=-1): - - super().__init__(threshold=threshold,n_components=n_components,n_stdev=n_stdev, - start_clip=start_clip,max_n=max_n) - - self._predictions = [] - self._labels = [] - self._scores = [] - self.roll_window = 100 - self.metric = [float('nan') for i in range(18)] - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return outlier labels as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - _ = super().send_feedback(X,feature_names,reward,truth) - - # historical reconstruction errors and predictions - self._scores.append(self.score) - self._scores = flatten(self._scores) - self._predictions.append(self.prediction) - self._predictions = flatten(self._predictions) - - # target labels - self.label = truth - self._labels.append(self.label) - self._labels = flatten(self._labels) - - # performance metrics - scores = performance(self._labels,self._predictions,roll_window=self.roll_window) - stats = outlier_stats(self._labels,self._predictions,roll_window=self.roll_window) - - convert = flatten([scores,stats]) - metric = [] - for c in convert: # convert from np to native python type to jsonify - metric.append(np.asscalar(np.asarray(c))) - self.metric = metric - - return [] - - - def metrics(self): - """ Return custom metrics. - Printed with a delay of 1 prediction because the labels are returned in the feedback step. - """ - - if self.score.shape[0]>1: - raise ValueError('Metrics can only handle single observations.') - - if self.n==1: - pred = float('nan') - err = float('nan') - y_true = float('nan') - else: - pred = int(self._predictions[-1]) - err = self._scores[-1] - y_true = int(self.label[0]) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":pred} - outlier_score = {"type":"GAUGE","key":"outlier_score","value":err} - obs = {"type":"GAUGE","key":"observation","value":self.n - 1} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - label = {"type":"GAUGE","key":"label","value":y_true} - - accuracy_tot = {"type":"GAUGE","key":"accuracy_tot","value":self.metric[4]} - precision_tot = {"type":"GAUGE","key":"precision_tot","value":self.metric[5]} - recall_tot = {"type":"GAUGE","key":"recall_tot","value":self.metric[6]} - f1_score_tot = {"type":"GAUGE","key":"f1_tot","value":self.metric[7]} - f2_score_tot = {"type":"GAUGE","key":"f2_tot","value":self.metric[8]} - - accuracy_roll = {"type":"GAUGE","key":"accuracy_roll","value":self.metric[9]} - precision_roll = {"type":"GAUGE","key":"precision_roll","value":self.metric[10]} - recall_roll = {"type":"GAUGE","key":"recall_roll","value":self.metric[11]} - f1_score_roll = {"type":"GAUGE","key":"f1_roll","value":self.metric[12]} - f2_score_roll = {"type":"GAUGE","key":"f2_roll","value":self.metric[13]} - - true_negative = {"type":"GAUGE","key":"true_negative","value":self.metric[0]} - false_positive = {"type":"GAUGE","key":"false_positive","value":self.metric[1]} - false_negative = {"type":"GAUGE","key":"false_negative","value":self.metric[2]} - true_positive = {"type":"GAUGE","key":"true_positive","value":self.metric[3]} - - nb_outliers_roll = {"type":"GAUGE","key":"nb_outliers_roll","value":self.metric[14]} - nb_labels_roll = {"type":"GAUGE","key":"nb_labels_roll","value":self.metric[15]} - nb_outliers_tot = {"type":"GAUGE","key":"nb_outliers_tot","value":self.metric[16]} - nb_labels_tot = {"type":"GAUGE","key":"nb_labels_tot","value":self.metric[17]} - - return [is_outlier,outlier_score,obs,threshold,label, - accuracy_tot,precision_tot,recall_tot,f1_score_tot,f2_score_tot, - accuracy_roll,precision_roll,recall_roll,f1_score_roll,f2_score_roll, - true_negative,false_positive,false_negative,true_positive, - nb_outliers_roll,nb_labels_roll,nb_outliers_tot,nb_labels_tot] diff --git a/components/outlier-detection/mahalanobis/README.md b/components/outlier-detection/mahalanobis/README.md deleted file mode 100644 index bf95332b2c..0000000000 --- a/components/outlier-detection/mahalanobis/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Mahalanobis Online Outlier Detector - -## Description - -[Anomaly or outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. - -The Mahalanobis online outlier detector aims to predict anomalies in tabular data. The algorithm calculates an outlier score, which is a measure of distance from the center of the features distribution ([Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance)). If this outlier score is higher than a user-defined threshold, the observation is flagged as an outlier. The algorithm is online, which means that it starts without knowledge about the distribution of the features and learns as requests arrive. Consequently you should expect the output to be bad at the start and to improve over time. - -## Implementation - -The algorithm is implemented in the ```CoreOutlierMahalanobis``` class and a detailed explanation of the implementation and usage of the algorithm to spot anomalies can be found in the [mahalanobis doc](./doc.ipynb). - -## Running on Seldon - -An end-to-end example running a Mahalanobis outlier detector on GCP or Minikube using Seldon to identify computer network intrusions is available [here](./outlier_mahalanobis.ipynb). - -Docker images to use the generic Mahalanobis outlier detector as a model or transformer can be found on Docker Hub: -* [seldonio/outlier-mahalanobis-model](https://hub.docker.com/r/seldonio/outlier-mahalanobis-model) -* [seldonio/outlier-mahalanobis-transformer](https://hub.docker.com/r/seldonio/outlier-mahalanobis-transformer) - -A model docker image specific for the demo is also available: -* [seldonio/outlier-mahalanobis-model-demo](https://hub.docker.com/r/seldonio/outlier-mahalanobis-model-demo) \ No newline at end of file diff --git a/components/outlier-detection/mahalanobis/__init__.py b/components/outlier-detection/mahalanobis/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/mahalanobis/doc.ipynb b/components/outlier-detection/mahalanobis/doc.ipynb deleted file mode 100644 index 76c608a5c6..0000000000 --- a/components/outlier-detection/mahalanobis/doc.ipynb +++ /dev/null @@ -1,350 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Mahalanobis Outlier Algorithm Documentation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The aim of the document is to explain and document the algorithm used in Seldon's Mahalanobis Online Outlier Detector.\n", - "\n", - "In the first part we give a high level overview of the algorithm, then we explain the implementation in detail." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Overview" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Outlier detection has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The available data is typically unlabeled and detection needs to be done in real-time. The outlier detector can be used as a standalone algorithm, or to detect anomalies in the input data of another predictive model.\n", - "\n", - "The Mahalanobis outlier detection algorithm calculates an outlier score, which is a measure of distance from the center of the features distribution (Mahalanobis distance). If this outlier score is higher than a user-defined threshold, the observation is flagged as an outlier. The algorithm is online, which means that it starts without knowledge about the distribution of the features and learns as requests arrive. Consequently you should expect the output to be bad at the start and to improve over time.\n", - "\n", - "As observations arrive, the algorithm will:\n", - "- Keep track and update the mean and sample covariance matrix of the dataset\n", - "- Apply a principal component analysis using these moments and project the new observations on the first 3 principal components (default value, can be changed)\n", - "- Compute the Mahalanobis distance from these projections to the projected mean\n", - "- Predict that the observation is an outlier if the Mahalanobis distance is larger than the threshold level" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Implementation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The core of the algorithm is using efficient and numerically stable streaming techniques to keep track of the mean and covariance matrix as new points are observed. The PCA is done by finding the eigenvectors of the covariance matrix using a function implemented in scipy. We also use an efficient algorithm to avoid inverting a new covariance matrix for each point in the batch." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Object state" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier detector class has 9 attributes:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - "class CoreMahalanobis(object):\n", - " def __init__(self,threshold=25,n_components=3,n_stdev=3,start_clip=50,max_n=-1):\n", - " \n", - " self.threshold = threshold\n", - " self.n_components = n_components\n", - " self.max_n = max_n\n", - " self.n_stdev = n_stdev\n", - " self.start_clip = start_clip\n", - " \n", - " self.clip = None\n", - " self.mean = 0\n", - " self.C = 0\n", - " self.n = 0\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- ***threshold***: If the Mahalanobis distance for an observation is above the threshold, the observation is classified as an outlier.\n", - "- ***n_components***: Number of principal components used for projection of the features.\n", - "- ***max_n***: Used to make the algorithm non stationary by capping the number of observations to max_n. When specified, the algorithm will behave like if it had seen at most max_n points, thus adapting faster to changes in the underlying distribution. Turned off (set to -1) by default.\n", - "- ***n_stdev***: Number of standard deviations away from the mean for each feature beyond which the feature's value is clipped before updating the mean and covariance matrix.\n", - "- ***start_clip***: Number of observations before feature-wise clipping is applied.\n", - "- ***clip***: List with lower and upper values for each feature beyond which clipping is applied after start_clip observations. Initiated with None.\n", - "- ***mean***: Online mean of the observed values.\n", - "- ***C***: Online covariance matrix of the observed values.\n", - "- ***n***: Number of observations so far." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### First Step: Tracking the mean and covariance matrix" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - "def _get_preds(self,X):\n", - " \"\"\" Detect outliers using the Mahalanobis distance threshold. \n", - "\n", - " Parameters\n", - " ----------\n", - " X : array-like\n", - " \"\"\"\n", - "\n", - " nb = X.shape[0] # batch size\n", - " p = X.shape[1] # number of features\n", - " n_components = min(self.n_components,p)\n", - " if self.max_n>0:\n", - " n = min(self.n,self.max_n) # n can never be above max_n\n", - " else:\n", - " n = self.n\n", - "\n", - " # Clip X\n", - " if self.n > self.start_clip:\n", - " Xclip = np.clip(X,self.clip[0],self.clip[1])\n", - " else:\n", - " Xclip = X\n", - "\n", - " # Tracking the mean and covariance matrix\n", - " roll_partial_means = Xclip.cumsum(axis=0)/(np.arange(nb)+1).reshape((nb,1))\n", - " coefs = (np.arange(nb)+1.)/(np.arange(nb)+n+1.)\n", - " new_means = self.mean + coefs.reshape((nb,1))*(roll_partial_means-self.mean)\n", - " new_means_offset = np.empty_like(new_means)\n", - " new_means_offset[0] = self.mean\n", - " new_means_offset[1:] = new_means[:-1]\n", - "\n", - " coefs = ((n+np.arange(nb))/(n+np.arange(nb)+1.)).reshape((nb,1,1))\n", - " B = coefs*np.matmul((Xclip - new_means_offset)[:,:,None],(Xclip - new_means_offset)[:,None,:])\n", - " cov_batch = (n-1.)/(n+max(1,nb-1.))*self.C + 1./(n+max(1,nb-1.))*B.sum(axis=0)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we have implemented a numerically stable algorithm for updating the mean and covariance matrix based on the following formulas.\n", - "\n", - "**Batch Online Mean**\n", - "\n", - "Let $\\bar{X}_n = \\frac{1}{n} \\sum_{k=1}^n{X_k} $ the rolling mean of $ (X_n)_n $\n", - "\n", - "Let $\\bar{X}_{n,N} = \\frac{1}{N-n} \\sum_{k=n+1}^N{X_k} $ the batch mean of $X_n$ between $n$ and $N$\n", - "\n", - "Then we have: \n", - "\n", - "$ \\bar{X}_{n+b} = \\bar{X}_n + \\frac{b}{n+b}(\\bar{X}_{n,n+b} - \\bar{X}_n) $ $ (1) $\n", - "\n", - "**Batch Online Covariance Matrix**\n", - "\n", - "Let $C_n = \\frac{1}{n-1} \\sum_{k=1}^n{(X_k - \\bar{X}_n)(X_k - \\bar{X}_n)^t} $ the rolling sample covariance matrix of $ (X_n)_n $\n", - "\n", - "Then we have:\n", - "\n", - "$ C_{n+b} = \\frac{n-1}{n+b-1}C_{n} + \\frac{1}{n+b-1}\\sum^{b-1}_{i=0}\\frac{n+i}{n+i+1}(X_{n+i+1}-\\bar{X}_{n+i})(X_{n+i+1}-\\bar{X}_{n+i})^t $ $ (2) $\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The online mean and covariance matrix are updated with clipped values of new observations. As a result, we can limit the impact of outliers on the estimated mean and covariance matrix. This can be particularly helpful when outliers arrive in sequences instead of uniformly distributed over time. Clipping is applied to each feature that has a value beyond the lower or upper boundary defined by the *n_stdev* hyperparameter. \n", - "\n", - "The 2 figures below illustrate the impact of clipping on the detection of computer network intrusions by showing the outlier score per observation for a sequence of network data. Scores above the red threshold line are classified as outliers by the algorithm. Please check out the [case study](./outlier_mahalanobis.ipynb) for more information regarding the dataset. During the first 500 observations, the fraction of anomalies is set at 2%. We then increase the amount of network intrusions temporarily to 20% over the next 500 observations before settling at an anomaly rate of 5%. No clipping is applied in the first figure, while figure 2 clips observations 3 standard deviations away from the online mean of each feature. The higher fraction of outliers is quickly incorporated in the unclipped covariance matrix. Consequently, the Mahalanobis distances of both outliers and inliers become similar and it is hard to spot anomalies. The covariance matrix in the clipped outlier detector is less impacted by the anomalies. As a result, the Mahalanobis distance of outliers is much larger than for normal data and less affected by the temporary spike in outliers." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![outliers_unclipped](images/outliers_no_clipping.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![outliers_clipped](images/outliers_3stdev_clipped.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Second Step: PCA and projection" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - " # PCA\n", - " eigvals, eigvects = eigh(cov_batch,eigvals=(p-n_components,p-1))\n", - "\n", - " # Projections\n", - " proj_x = np.matmul(X,eigvects)\n", - " proj_x_clip = np.matmul(Xclip,eigvects)\n", - " proj_means = np.matmul(new_means_offset,eigvects)\n", - " if type(self.C) == int and self.C == 0:\n", - " proj_cov = np.diag(np.zeros(n_components))\n", - " else:\n", - " proj_cov = np.matmul(eigvects.transpose(),np.matmul(self.C,eigvects))\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We compute the first principal components: these are the eigenvectors of the sample covariance matrix associated to the largest eigenvalues. For this we use the function [eigh](https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.eigh.html) from ```scipy.linalg```.\n", - "\n", - "We then project the new, both original and clipped, observations, the rolling means and the previous covariance matrix on the principal components subspace." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Third Step: Outlier detection in the Principal Components Subspace" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Substep: Fast computation of the inverses of the covariance matrices" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To compute the outlier score of each point in the new batch, we need the inverse of the covariance matrix of all the points up to this one. This means inverting $b$ matrices. We made this operation faster by leveraging the fact that each covariance matrix is a rank one update of the previous one. \n", - "Knowing this we can use the following theorem.\n", - "\n", - "**Theorem:**\n", - "\n", - "if $A$ and $A+B$ are invertible and $rank(B) = 1$ then\n", - "\n", - "$ (A + B)^{-1} = A^{-1} - \\frac{1}{1+trace(BA^{-1})}A^{-1}BA^{-1} $\n", - "\n", - "The implementation is:\n", - "\n", - "```python\n", - " coefs = (1./(n+np.arange(nb)+1.)).reshape((nb,1,1))\n", - " B = coefs*np.matmul((proj_x_clip - proj_means)[:,:,None],(proj_x_clip - proj_means)[:,None,:])\n", - "\n", - " all_C_inv = np.zeros_like(B)\n", - " c_inv = None\n", - " _EPSILON = 1e-8\n", - "\n", - " for i, b in enumerate(B):\n", - " if c_inv is None:\n", - " if abs(np.linalg.det(proj_cov)) > _EPSILON:\n", - " c_inv = np.linalg.inv(proj_cov)\n", - " all_C_inv[i] = c_inv\n", - " continue\n", - " else:\n", - " if n + i == 0:\n", - " continue\n", - " proj_cov = (n + i -1. )/(n + i)*proj_cov + b\n", - " continue\n", - " else:\n", - " c_inv = (n + i - 1.)/float(n + i - 2.)*all_C_inv[i-1]\n", - " BC1 = np.matmul(B[i-1],c_inv)\n", - " all_C_inv[i] = c_inv - 1./(1.+np.trace(BC1))*np.matmul(c_inv,BC1)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Finally, we update the attributes including the clip ranges, compute the outlier scores and return the outlier predictions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - " self.mean = new_means[-1]\n", - " self.C = cov_batch\n", - " stdev = np.sqrt(np.diag(cov_batch))\n", - " self.n += nb\n", - " if self.n > self.start_clip:\n", - " self.clip = [self.mean-self.n_stdev*stdev,self.mean+self.n_stdev*stdev]\n", - "\n", - " # Outlier scores and predictions\n", - " x_diff = proj_x-proj_means\n", - " self.score = np.matmul(x_diff[:,None,:],np.matmul(all_C_inv,x_diff[:,:,None])).reshape(nb)\n", - " self.prediction = np.array([1 if s > self.threshold else 0 for s in self.score]).astype(int)\n", - " \n", - " return self.prediction\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier score is the Mahalanobis distance in the Principal Components Subspace.\n", - "\n", - "$ score_n = (X_n - \\bar{X}_{n-1})^tC_{n-1}^{-1}(X_n - \\bar{X}_{n-1}) $" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/components/outlier-detection/mahalanobis/images/outliers_3stdev_clipped.png b/components/outlier-detection/mahalanobis/images/outliers_3stdev_clipped.png deleted file mode 100644 index b3ba22410853844f36a80055f0fd6bf888ece285..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15130 zcmch8cT^MKw{Jr47^vQ}1R=H$#dv(IOrz4vGDiP6@4Oi9K<1_FU7RaM}+AP_he z_%e_X0aq$R$({f|1l|u-^+|wF5Q$wha8Bx>V(bk9k<9kgK(ihl{(9i=)l$=k{LSj_z)^MTJC##O~gH=HufbBP{&C{~+Y<;z(-5qz3&%=YEPw$j{5qi_MSE51of7 z9A)mBxFZZFOIs%dbrA6)JsEa=sz^NGU`PIL36pKSDhoM(50mYio)@t|W8tR)n|Kj! z$A33bs=!-fqhnR_aM!P*<5iF?qN&ky+|bY%k&wdZw(Q7bebdC~og2wYZ>s0w;H%#* zgo;IIlO3U6{xs>QZ=9aA$GfppY3Z=(h5z zHN-ImjV)7Nop`o_IMzS}%Se~bUzTjmqXMKk_Xv`_+XeM~?P=1)R1s0!98Y4Rd49z9 zwL@zj!5d=wHlU&niQBRM@vPdDNd05k0}a;AM!m^0{bR{eRr5n*?U;vfH@pz)If^xife9c;N&sdI>d4#zbPq3tiOm&U(|6I4!P=xFo;CHv!9rvA?2p z2L}zOg|ITgn0Qa}^9lLG=0l7dLFQn>L#+6C*dYE<5?&f;5riY8qKenUuvjry z^FPcY59`Fs4iiLQqOU{DWUZjpQ^J?|;pdg0H2myv!*G1t3Rt5<6^m0Iebd|QQvKAh z8FsVatC#~Gt4DRdc;CV$&>s~Ng0HZTFE*#aJZB@W;T^z2h@qccAK6|LTsm6KJ;R+- zp9VBeMhd;e9LwbVgZPW5 z2je$ySm*Yh)7~9Rcvw%HnWeaPULCAYb(1dRBp!WM7TyTp`V4)_7$$Okq>m*W2!%-m z1v<|ZlZSglD;m?z36PSZJl{nJ#5;+u>1TPYR}5$w^uP2p9{e1YsfOWbm|zrV6QOKW zLsaUfBH?pEKWat@86rR0>tilsBIoTe!^zJS?fE?rn9J|q-9bA^gID6@@i>wF=a^Fy zW+q8mrd=2{w}-jPpT-XX+A$kB3D4u@BU)qP{El9i!^-qxcATvidDm)gU7gv^3f^(dmb*l z2@1K4VB)g_7}}gufiZa#>Sgw}PLo*kMl!yKbaW?v1iVn7GTPcY{x?OV!s~1EYuh-I z@mmBzl_-1CyRwB|>}SDs%#C|a@x8xCFijrY`1wek)G7R~G}`UPNAOp1eP7Uf-)WJ> z`8RwSFtJX%@uD-qO=f?xp*6bjp06r~mC+VA>quJ$I;;qWa)J0`G7{uQK z-tlMfT0o*vx@nP$05~Q+&-wKwGy+pcyy7LhjdzLFF-s&gd*H+k?TzjG{JGz2!zC zWbrKb{SG0wpsWr9K>QBKrm-P^`ZvYiqST6rs`rj|7s3cN07R|euZD!eK0$=^*7 z)ITm(pxi6ll5V2?o!``2sN!gzDEp(T%YMFbMKe;s6X{5w{4}oNx2FldPpK{8uHrjt z=op@%c+LGWKeJ>d53{zZrPvOrU@c4NXuwUwy)g^KPZ0Zx5R-OeaB5UpZHXeHv25!J zDu85P9`gty%{9;>L@@k$zPYhy?H*~NV4AP~;`GDgt9{wdjt>(O_nM@>ctva)6Z;$v zl5W!MU33+hN3?S9uu7enmTo13t}}wc>4D8R{JybIPjW6b#sPp*K}?rz(W3%xSHhR> zK&0y$&S~rwx$qG4v8~RRDbYE1py$^@7s_VOvBt2?hgKQUbw|i~$w)yb{2O~mR7oHh zV=@OAURWQU!|tA?Y!@B_?zm)CO18;c93vHy9Fota_7p8QAn8Im=d(lS2{eVObxMzg~UQx~J=;j4vcY-krf#Bvq(Shs31@$1Np8q}*JI?~Oo<6tw| zH*qS4FqK#T*aCCk%=xuzlL5bTn4Z3GQvQyy+1j4d9GEdg#&&6@FNHkV!g5XgKtEMd&kOrj}>LwA=o65cwI`zUc0YpOj6vAF(v93}gzU z=S3<`9o<-rMx64m%RG>8Mm2iPIiJGM**bR3(dKB4P8ji5qvwU^gPzElsO7DYSWm_< zKP+X_EyJBjYUC)%1>-$-WXgd|<(#Ze(n{jHq%cBss0ZU$7M^xg$O~t_`{YQ-)=Pmd z>P)h`L{o7pb?;|!?@)wemY(z-p|PUBt(#uq%VK5NbtfG|%+5rvDJuQe)y=vhX6jv# z9<3cc2%E(Sa0wUJ<2LKr2&jR)~3 zGwi>J<6WVb8e-h!@@{v)ODfs=T(9Y(LG&pnR)qNhJS(Q}o~6b(X}Vaf(WtWqx4N-5 zB$-EfRRD4>b+0@(T7x0k{yNkYW(K!wi@wX4oCGg)x5|TrT-ek6;!%fR#B4vbZ_by@ zi+x}#nu>`6UHNE-5cR|Cs;L`XpCE57Rhrj7oVP3Ci);lqGA>DE&3g zFvL$}GT-iZd>|EcYJV8=iWLX}KD?shufluBV8-D=o$&G%^=So~2xJ;8G7}p9?z(@m zp^13!db`(^F&W%3IlYgr0iuUO)q&Vf_^PzoUe0B#ib&=Bl{}gp>czZUx~?tBiThlS zB|9edRrq^9czR z6&7;NtIi0}(b0Wx_I5ZtIx3@NOg=}M(2ajmoGBHy)BL+J$4mu-q6f8y1Rq9Na2wuzkg24PCF=eGo?)4E#TA=J^4sT z4=k6eEpW+pOA+3*S(MG+xfqJOn)B?vMVa*~nd9-{n7XjUFPRVjL_^2l>m2ng{MYC^ zu4QH~zuO212&j9-jt!cGGTX8w)EtssOZoHdRs8shcKh9YP`A6fHnq0^#C=HuRc&uP zS9(0B&5;P!)ANmdF{cUM`)+e8QB=1wq&%VKEk4$(eaUss(}{%S4Bs2wElnC+`ca57 zqeiz|>ghsjNugMdT8EMK1j&U0y<$!5Rn? zYWW4NY~5`c@C3%QA9%}$U7`*JS#`(Kw_RUsW4A4F%gb@XPy1fqxoc7v{q$Sb!Nqo! zq0rvd>Egyp6725`+D>MjRaHYpjkQI3#9072z*QXyL0li{hs-|l@eR68qUgJ#s2#9p z2a~TxvLxcjh1+g~sS7nnA{T=jz2*S1Xt}w$t(}UE$faPhK>^2G%KrY+4mqAgp!R@c zo8zrXP}|kX&zpfepZ;u3#+B+7*Z3VQ7Z$=Y{|2cn3E}gzdaw6{0twHhDYm8*90xm= zUa|Tu$MF0B5H#WVLKy+*PSyQN$MGW2>gww2{CrDkS4~YSdwcu0Z{Kq z$m2p;=D<{yOYvkXu8K22eKf|50GTtTh%@K|8p)aPUvT~3nu^$Bs=Izo zRj^K5hRQF7T)GbV?{-}7IxU*kJp-d>>z$?d+KzP7re0b7qZ%rR^Xt>jk>-uO!5aHP zqZS`N+2=b}J3l^&Y>-p)-yvr6vwEmCke;5-^LF8%F5`Z0(?7qJT{_U4!yR6<587=_ zlmeRUvH39_?{#+eV3X&H-(f$G|F3sZO#o;FisiS-TLVb?vpb(1C)83h zduUK)@nj@V-duNZu=#vFdvmc9_V|`BU6R5DFKE2zi7kLfGry&m03OOLHnj-1i@KS{ z{n~;Vhy0n;kX^63!_cLjsH0^DVGE7UkSi3WGXO$B+hQ&bD*|_$qeyAFt38&!WDJG? zFnKoLBC*$rQwTiSvHo75NO4PsF09#m z0fL$cjt)+N*?o8$FukBZqmNU&txX%<>}P81#T~~YZ4;%rIq=E;E{{0(OP-dSYBO$q z&O|SHCY?PWv-xuhcVfNjt$#MS+J4KDsB@doij9pOSc-YBSB1iEYK1QU8P}@19|T}R zqUGBNr@>#vmU|Wd5^Shc3_T6)b3Hbh<15bjQfc1mx1Q+$GNkqzelTy?TQ| z$V^39nb3Kv+0+zw{F{4-5^&3|SClKet$RhPDcp1n3}a=6 z!T~2cs*4@L0voM|0|L@pB_DqMY!n96A+~8}P`EQ^vdnNC*w*w@scy;akAz-}IXY_5 zZ(k^+JZY{)>55ckw$`dGlUi#u)K>1LjI?wtW^*;`jxyQmetV(DEg(0#rhS_)H5P!l zlfi@h{#WsYU)QwBv{1;8?*R*xMVP9QwyLHK7%&NeYterxg2wWjxzINvPxq`9yE%7% zw_2Sj?XSuCA#ZBPjpM~RzP;}{3*TF4-<)riB);hZ%9ZhbkK9|Ra~yxgCLh2V91Ngq zrMz}n?OERE2x8hzk=WwA0F+5odjFfW0x0x@vfZcXpu5nK7?tp7y4b#(5MzJ~V@&1_ znJv1$^n!IbTS>!CyI{1*;~y$UIA>7I+6-G>58s^y`^kD4F7)ht1MJesmzU(mO&)g) z&3)-VJWi^_=B4Oeu1S~_r_7Xpj);x+!K=iN)vxDyO>PFlvN08-qQHxw^mAh&{0^N- zC4g#>ffq*Q@2OxYFn1vRu6?eKb!1uuFLKvP<&|tTG7b;DfQ^hSQSMOX71gbQ-TB?j zTHsl{f4Vk$j;Zn-U?Qk%JXaot>_@PF?+gvSINci=$`k`m?R05dk8O*5W`&~^m2Ec^gbG& zS^U`zF^A#8l9K4ZrhdxQeY8Cg`HGr-xQiM#L@7C^otfvmJ>_}%yHPCkI>@Ba?c3`+ zO25X7wdPNs&{5{;=}ZcTUxo{Ne6|y~$Nu9z>Rp?a$oKxxU4rt1qqOtQpTtp*Gm2Rt z&ARaQn7+0IR!MGT1j~mxxu!pqQrA6&zC#W}3q{q{uNT?_@xNDD!hqe*TzdC0>_TU4 zgHNx>1F+9ytw{G_g-e$P*PlIr2lIEzk_&zuE4@h$fb+Au)+~n>>Ak76F`V1q-~SFk zaJfZ$)`-Go{Fg5eTabHH=x7K>hKTK>`ai(hMghw|{!WQFOWaXckg|UQBTWx+j2pD4 z!!r$I3>V|jU;&(mF?CY$p8YI*1Uo(({`9Hg3)TC{Kg7hjX(u@xl3ku;-><;tWM`Wi zV+r-QTRPsqe_>qvbhO;K*6ApTS?twb$Z0Ef1BSCkhk-RRMY)!}1_uVB8>xFQI##&Y zHcLui^hO_1tcZF#KSCywI!Lni>urkih4+6nltNnPJt_^TRce+~rPfj*sp!bkjCT zJ`FJPpB%@z0dLYDK#E?q#3m`fB?upG$8IUS}a)$(5ZKfYtopenN3}mzqogJ)XO1iWCwLbCze=y09*}#}^VzvSR z{WpXEjEXx?#fy!^>lds5hVGu{GJF_@`^SX=91K9e5%rET&v*K6A?H2=zGZJ|ah^$q zz_p#4`t&K(i)WvsfWio5mRLsZ&-QF1%DfyE2rmTrG5m@cd}mgG61ei}!TW&qa?_ra zbTUO6_y$u8MIGATZqJRy#5Rtlv)PJ9`v&I+9sj;i5Ul9bKYormss1eBS<(2VSP(p% zRFl&h*7_z7t?_1cI-kW>gWovnP3CX_|NS022swY|u!%7W>w zl+N`}Sso}Vf&r&mZJ@jQlO6)tF!j%0+nXC}b{9S>nKapSGvtXhN*v@g2fzoo>^*Ew znu&mh$sl*ls_X%aSNe~k#IP_MEn0CC0xmq?-Mbn{QH?PA(98*5xzTz9d|(s}4jXCu z6qYHcZyAsJU&Q`1vc?CyZcA?2!uJxH_1a?Gn}9=Gqa;_+KQK7wQ}(n;Cijl5d19%; zpm-1G*B7zf$Pq!rto<)gClAgTXo2!5`p$Cfa0RsGS>H&Nc9C5#n^x0{d!P z`Jo_C&v9h0R)dX*Esb5H4Y%yq@vOU}!;(saIY%UKcgq#rdSy;e*YJ#_@f+w{m(t=y zxVdG;G@eHnOBnlMq&FGT$Bm?K!L7DgBo&yLG_U9jDu1aDa@?)MncVzeiYwe@F)wg+=N6VW_oz5hTt&i@;zRKB1p3vADI9ry6yGA&y?;UR}`;4l#4><@jjQc!u4wEeCIPCCOzr{IUU>$kDL3pxM11+ zjkh2BGJ2eFY@3?lBF79`+R_GyDzF+z|CqHvRZY)c!&ZD*V)~;xm!~lG zxPG#OiPE>Rkr!zo$Rrvh}KQ@442@-UZ8ZH;+(ETyMELW6I2Oe`!4N zHF$>yiUjZBo!F4r>fV{)FZXpg)wa`~aWmRC9IVKQH*nV0nN&p!qW1u+-i*4i2{zz| zy24kvAm{qcsuxWL@oxCfp9yD?*<5W`ebV~4zDZB zHk=r7Yy@#69>S0HQ1(6w59t`@LA`$u&6RYw5S8rBSQQZV?zZw~3m?ms=tXW(zvMx` zP(^gxVz%r|kcZUziS-U<8oJSaK?uQEhuMtBIVgkZz9{IJGNMyOci9BYgU4|t(?Qrl zpX?p>Ii#jI+!0-YmPU(hy>TdaspR={0jL-sz1G6nFkN}A@O~#s<}sX&pyz9b1(af(smqy4KbXRg z!tB7%reA_P!RyG(&32#3?;^00_3A1qQV8E_cct=BA#2RsE$_QP0{E!{h+TpDu_dOX zDr96Jf(ZI!AcxFksRLtA-v*z740SrCr&=oBFYMG0 z3|cK`BBWE=o_ahG=FMZpiJIJeeSr}@f6h2jH+SrMf4}9=#1)*(jo!YJx`8skiH+(} z+d5tVN4kn~u$pfco|Gj*GAyf(O8m5l$&9fZlH7gSdYt{F)aouhxusLyhXJGqI=vb?TkWH>-YpAc{HweVy{I?e1 zrPZA`rm1?QJk8(Rln@@_s5tlOmHE>Py6kye?$)NGCkVS!OVB7nDxJ%R{E`!8KCf>f z#^fPUp0^ZHlfHN}@k;gSYSCCmm%7QeurN<~lQZoBV*l|+6uIY<@UC1MYZJoCn3^1( zaGvcH2HO+<_!6HrB&Uv ztmiM9X{D!^-cA=vzAYjYHr_t#V!ZxaAUg$<>ec^@pN7=FyrT6m8oU$YY zje%XuEgwt4Uu$3Dc%ZnCsM6h}FiG8XPFoG#m(#uaAMQCnT)nCO>mfOU(NeGq^`1{S zTo@mDT+7{Lrg@J*nul`o<=+`E>R(hlsk}s=!1Z z63#c5(R2i2+7*h*aI$b*$(F!%pcrHmg(}m_m9o`{MKGkFlN9 zdh2B_y+S4CaR0(gH6fb5~p=OnS;y{05 z1fS@FXYpVjHKadzE#fewZJ^_>@dn^lCKe6aRd#-{clCFAEicr4zLV28`}sW7e9!89)-SR1a{>-#vA?;gZY>?5C8 zkx1X0;=+U&L%I2i$d(LtA`SgXYW%Kvk>g}8Emc<9D!&&4K@fjhM*V96s9<>P1ovRZ zBj}mX1x1M3Y=&RA9@dxnJF8JCsuf-+Ur92=q-SY-__j{u0GOX4;jTH?yS;1=d3sjc zEob2%&ClX)$6H?(m}(iT8J=;+h*rTswi;bEe~)d}_2Hb4;Rd19l-U{!AiZY(+%Y#q>&j)nv;~k~`)ugyK{I>WDQ;NdB zPM^dU$MtUl9N63J28ustwMMR8ftx`TO^If+P z0kJQT=><$<=U+L~=1PF?GvCdNVn2;s(4xK#^?$H~yuoU#+nWW-rT{E|rVz9d{0+V; zj#0)emOe=n;pTV7a}hNHW@+n@t0sgOf`=tOQ|G6(kB|BMPapTFH%XT;Ta6a3F@mPH zp~=tXSxa70Nu3Q#b*WMz1XlFsA@-7HaCjVpW8j~#@c%Ne()TNZGz zvO6?3@v7l2WfLO}x1Tvhv*_#l=HhVvU0l6PDf$4b73V#(yHoNXtq5#Akd)%m7rv%> zfT%~rjz5hrt@?N0R6WO;*Q5Yx^Pu5)JJnV3`-HYLWk$4I87h6^{t&>!3;ujjPn}EB|1_yU=E6%>%f_(IUCBU1IcI+T_kS6lBYRZLT`{qIHHc%+Kzv;9ABuAOE4a;BYTqPgxDsawC`hhifJmDei={G$MzYc4N)>MDXA}o zCWB2zNBZm~7a1sqQYRfC*Fd9I1*DnIx{q=O@=IEbHbCv+)Z3qi711+LQ3_s=$�l zE1Yv=!$1Zw-G92#iVr@y56`juxXHvulhX@GiREjGUMBI-X=rVjDdKp!rBWLaT8YX< z)C*L;4@RLZ2-+xfHh5SS5w&ZES4iwxDGC*GM%dwNbjO3~USs7Zz|HA}9{l0ELSnSd z-0WYX(h7M68}UZs`rN|_(X=6FwCGxZ4@y^R20cJ9#`Te-c^)u(B%soYd0l}AUB10r z1p7i_IN>9CkL>Jbdf6t020;(X$x(?*G!tM>eHPmMH#J`YaG```R-u%E0^F&xWrnId{X509Fg-rzOw=P*Jk!9Ha2-2V$5|mfL?P zS))-d2@4^;DFEeJk#VKNP$GiM-qAP!#dNJ-$4{sAt(@=+0o-V$m8maU0{_>5|36s! z{|ll2|Emd*YFJD~dD{z^ro}*@zRh8wj)1SaLX4q6p~q?ftnGiu)f)dtl%V9UuH@(5&8?;; z00M4kwI2J#Q8QVj2cDkj^;Q(ajgZG<0-13D$|!<1MFlYJr{SnPxIHp(tu%g2W;gKM zh!Y7D!EV%q@7w!3EifCg+BW@7A*(*#76#{m(n_}2@jqA)J``+v1j3O~ zuzlE+$=4 z-=i?1+QAbd)SmnXJ!HiM!{)1$QcnQPjwzXaK=phmDN)(@mYu$92cUTB#=p|DbyJH` zz(oF!`ca_Q|5RG~PtyYSnP_7B!1e3E;2j3XR@|Y|;;$C@|aB zM5OX%yp5ydwE1Mn%bL;-5CXbxR*fe?Eg`HxrqPp-H1K7O$jl%>BsIe$BL98A9Ksmh zfk6zA2AHJam4g5&6f;?_C(mm+eq{LgUkYU$-Fv(HiZTuyt{0lRtIw<_{MXZJ0W2jR z*#jW@m5^`W6yYWX@{B1$FL@qXh@DDTG1KL}UHHS4LSTTk*Jz<&CXXI`N0&#S_{8vn8YaAYz4Ccl6LxV0kJH_@y2xhmwj8jB3J8e7KNs ztp}PdH1^tD^3DuH^?3gaps!QyKuZq%v#Vn~^7-b%x=V|S()0O9J5#hK4>>>g`-?ny z!ojf6{Lh>xjE;SCtSFTlL95~v!c;o-(|Kp}$x}|Je|7Y+EZp`kcewC>;|i7qBv@$F zvS+c4s^ro6wOsvt(9O#`@{p{8LM=07sidkWM3Pl*ijm%Z>En5K&UBTxS*JI59#3-O z^k{K&UEXP5dHW%gCXvphPw_ho_$mcP@iyaG9BP5q`Qkt5gNVYv``k*%DFZ7q$W&S* zEZ9_h2|e}1_+nC1Z7|g$*W_T19-sHw8$|9Qh%;8N8Hv%Svtw^F(^u8Zs_P;#o=e0qP5%hv9|{LH$Ly0(noP<`kaHVWpu%Ce1N73;YJ%^iy+gS_i61 zKfb9g7W{Ycm0}@>^}{L%BCUL}C(r9fxB_tl=H;$l|K+kduUv9XO33h4isAktare$b zZB+K5h0QMo_01oFywW_lFYV);`+*68N3>+HfY&Rlw)$Q_j1L9lj*{NvF=ZT)-uij} zvA}>)YsFhZS#>|lfKMHZ2`ZWjwFYh|ib8y9$0R&&gP`v2)@!wqv6n6jdr}t^o$16y zmCI|?u`?FBvC-daZZBRH6^BN(&E+`tK6sWNt+FQmUPOcA}zqqt|VBPAp(c9ck zT`pJja6|i-923mfe<(Z4ip4uj3;X2ZxZF}-Vg=tqP3x>kTj*fX&F?L@w+;sZE!#hk z4r;Lutk^Iq+D*c}6RzL=va!fm6?-qF0_ff=OJ) zgL$W@e9{tpURSK@Xi~*4yWu@_Q2FJ(AL_A)d|d%@nV3qwJiUD$o472bO^=pwwRANT zm|<@Cy0uAqZSi7^W5fI8U8JZNA6HV)KwmKhR}|%Jny^o2#x{kFsDp&y5A)_*pRXXU zfvA%sBC6MdBmR*x0$}*6Kj!9bCv8WSQZil))ep1K=0WAn+ot^=bIjfb7ZSW4h~v}m zEO^Ay*+{Y|Tw%v^_M3uz`r#a*FFW5m`qm$h)%bhyQ`7q}A4P+Vo-JqWGl3WH2R{37ZGFYZGm%i8X=b0-L5Sw^ z{j1SA9CZS@o7S@he0%S;Glrd_sGwn+-z?k3rFX_eNV8IV9X(952HhU?#Z*&%8t#G* zkCGu2jA~!nwlp}u{;dvgAWqdpXQ5ARFhBNV{_)CjzEROZQy|9={otq*^~svkV$w@B zQfYl{MXEL4^T@mcgXfh!G@H64Y?H`R+AW_|&4dx!^o(qDy)e{|nJjAn~e^h6dWpe zWB@5|6UNu2%cCf%7p-TN)*z}a3)--dXR!@k^0sj%jv4@er&htX7u;I6`QXp zGy@GJ`R)VhdE=V)f67*f{{50k;p@Letobp0UJ&M#-9H2|eTvY-hUmMHo^P9lpT@@CX|Xjw+|xw}B#w;M?=&5#uif9fUk`7KZ4HN;Bx|^5`;(eiE8> zVX7hDWeVa@Ctx>%Yi?*oUyct?fQucxMX;n$vj=&H1A5*RCaQ|C1Zz)HqD(4L z^yn3R`ewVKlaehH98Lhk-X!iuLhpQzvpnw?``&7Z#_P#v>j0<9(o@qgGPBs&GE6d~3@1xe`a|knL zBxOn8)mf=XcGQ+btWuhCUbH~uYRRPgZudMm;R!O}%h+={A!jV4`a6yiKaY^%e5nfW z7XDHE@y|nMJ~v8Qj`=8zNqrE>U8>)P1@ik0)k8S_QCt@Xs=ig#shjZ zLC_utWzKG7Qi4X)x{;*dKaoLf=l`{+?VlKUoHtsfd2iDfD9Yi5O2(=Ttt_6}1X|-4 z{uYLgeMoa0PPm^CZN>%dqBYwPhw!P{vDq#sPgc?1=LWYT>LtUXcZji+&nV~EZ1XCC zVwKsqI+KGd89Qbe2}}yf5lq+H%XX~Gi@#`8XS#hbwQ2a`>xRk#B$*n!_yD#kk0w=! z5|ilxay{btv3=tm9O!hQ&;Yhex9GnFT~xHvgSClH--gP0UNUZos zOdq}K)tw;=AP{hTtMALTd$veD9X08=B~yxp+*{DPPvrL)5m>g>#5Df$l4?H##YX6Y zU&)Y+hvCCKM$M8Ns_l=H9Ni~=TDCO`yu?L@2&O)TAAeI26mXhplS9Z8HHD*cerSc8 zD?CAhuRw4+OAI5rG^BMMEQ#7Oz?fk8AkCmtwzs{`##JH=`T#w$H7UXFX9j0j3ZPCj z=!ER~IP3cs;jo5g=~9begVHTs7}jwvF7r!poWMF>Mv}k_^$7vUI&8F<5OGB^eN5bx z?sN=#8r%2KEZ>6oa+7hKdt@@~l7vpH79PzpJ|T?A>*S0F$WXZZH;No+VDIvbrE zGN*(lfn+8G=J#mAup9+LhFM98BIZq8AuUNB$TR9PCsGf-{rAEOWoW_%KY8l89f6D1 zNg3ZK)cAryS#F<^AF5ES;mK(4kU1BO2fL9Fv77EB(nE2R(buTT(XKq*#`cZMTv?{2 z-ejJmW~_*#we-Jgng9FG-}24>Rss5N!srMmFO67f!dY|y@3DeZl{Mj&53OGOFRmL< AkpKVy diff --git a/components/outlier-detection/mahalanobis/images/outliers_no_clipping.png b/components/outlier-detection/mahalanobis/images/outliers_no_clipping.png deleted file mode 100644 index 980c6ae77c76d7b3e4fbe1212bbb34cb286181b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18498 zcmcG$byQoyw?7I5E$(iilp@88YYDCeg0xtR6?bUkTHFcp zPQUlQd*55@_x^lYE9)fZOlIcn+49++*(Xt28jAQ&Xr7>;px`SjL9|g&P+`EE0vi)J zcEmx09z(;`$tzR$qUri%T)-077_MvhI+VDXwK5nCh|a8+S*RXJE?BN(?wcm>33#NS$3&j&15&b51Pjj)f4 zx@}2p`*$GDBy-_Z=COPy5?={Asm#@;EO5Gk!>bggSib(J|K~4iZBa_5#IK1@nvM_@ ziFN|?`Z}y&ub9Cc_|2UdvEi>YzJgbft>{7Z!+lQqjW*2m$^#ztFRU<%K}nQyio`H* zwK0T&G?Ab#tz8B(rgqr`&(4F7zf`LojfYEpgRg=d)LhjY(%V~fOZH;mzIt-K7D&}8 zOPt(~#EIg;2o#-=%g&#juRHCDe-1g@e{xM2Ip_Z*YGe@lK^Brvf9$r>p?g6OE!Ukv zTOrAO>XIzJTmwEWE#B>qQu>a(#rQ^eOh`P>1AgS`as|s1A0lZX8*aomdXQ!E#JpT7 z9pWZKNIQDF8gC^X@l6ekEM|~__yz)LZp!f>OSFbql!_k-=M`o=1=B(-8en(E5MSK} zHDYQeaLv+52T`JGZojlXM2a+#gm@kde$28!2EG8ZHCI-;rV9KD2+|ReB5u-zJp9!V zLfm%@EM@2AZe1m`m}EdUGK*Iw7d`$QRIKM0kMCV9HLYjyfH~zYa$xpkiBUNHQS6v* za~A{P(`49Rl0+=|UecF`uZVBjV1LOIJ;1IC)3p=w`g2i(p|-Pcmpk^J3r~3y`F^ej@T>@4gj%D%RuC%ei6u{nfqF3hp-n zUVQI2kly>htlx-sP>s+fhyps*pCun@Ks3;@2b#M7uJr3`O|=>$KoOvx;Ik0gTq)Y9 z8c8wG6Hdo<%I9toQbX1YEMUTv;K;a2loKd7-Pbh)dAQqwa;9 zR}&%1&4fPj;dTqOPO$4dSp+N0T=!)$d|1~b2j-47hFVy>*Kjzab&cZB%BurSEwvRr z)-n46J6EqBISWRE1}H#5@Mb!1{OuLmxOd-RwQi|(5A*0o-Vkh3X&MC2%}p(l>a z!Ee97F+;YUNsNVwckwgP7E9ok@Me;0Qhcv(t;E&~qk1z5aL3m(9_$)OTCa>ikVI_K zhaOK9!WP7osS8PG(9^G))gDKnSd_p*I61r-<=T&ab|QZeSi)-Doy`2jwicw1ocNRL zthW-iSikYHe+k%Tj?Sn(q8)P|SHJxts1CUk(I;zxUM!$bop(0bqJZ_4`ewM!o|{>S zCnhOjJ+cMxXc`o935<;H}pjes=B6IoDsj`huAGt&NmI7M1`^7Y9+tfSyqh8v_^k3r9Jo?n8&)%C&H_jZ07iRwxeN2dJ=2|v( z18kFSCBW+!YG!Um6-yK}gF9Gwoow74b~_B6kuMn;mt4PVf)n(zgT)CO`?H)Bl^1Vq zImcufVP&RlW~!ep9Wl51`hF`wgLe9lqu|-`I(O&Kj#WMz2%}(yFoMpO!md5d_^V`~ za#7Dz>kpOf`wWljU-#TX%HIZbw2aDhyy+*K73VH2e|_YqKu*efD?z^CJS*vY@5@Fz zKb-?OBM9_rO;`KT1oA8zhLCItATLrwVt7p_cQ!e)9w&B|yVP)?#c8VV2CHtv&fk3Zr%x4|}Il!P>X%qlv5^anqp z-^L8i9QxmZ`CmceNE5kS{@gX_&!`ATn7w2k2RW{4xlArLUY&TNF>5bo!<(rM(C>0Y z$gc<1$#C0t;FNPZCvPl;8?430ED#|#X0!yum}zy8tUz=IhFdPrXL zN7b6C3@PXgIK1M2ay|ZMq-Qz~xX1qTv{qVg>}|6v;xWT@YDXVQk*GDD%%PuGu>0>} zI`t3IjxK856q2up{Xj*A2gBfMLW3~3y3cTy5JShZoOQYnJ>_~8{mNAP&O?n(9;r7{ z#~mUU*bB~MFrLz9{Uj?~UYX@0*7qH#)p;p8EB(?K4R!h?XTkM!wpnD4Qi1bWA;^%A zhnEsf={}Lh1m_;+0iZ^GsC7%u%zlf`Onk_`3&-76^9dulvx=B`n;ei8j`cE7S;Us`5WikAXh?hUP_rMLp?mQ~ zkn5_lQw-YH@O|6i?~`sSH!7Eyhufw;Q&N}osdmFD_d>_X6edW11I+*V7{0ch{07_M zaopNs!g&Gm5!GR-NeK0jS75R2pVaH!sMk5F zfDbQGP!2mgYBUoRmHerc-{4-shPLTI#xNdmi$qtIR=nV1{4ctw&0|Ym|EXZo^>}+7 z0d62UcFU~)v%S7}7m9pDySljPJliusb9YVx0&N=JhcbqzNHjT@H~j8y8P9X!3y}O$ z@i&_9oqXBdBOPKg<}5NjH)|c2ZhKKd+72Psv3!_j^VOCux^=4R>jjdsFeTc|t%!zZ z_BCn9$Q__FXpjl@$)+>3_DBA z13ovC#)@sOmh}GT_tj~yw{Jpu`>B!=fK#ziVDO^L0Da;Wq2xKy)smNKUcd8_#~h#C zY?*?w75E?Sd8Xp&DOB0j*ZwoLWv>wmfYN^-?8igXT{B_}h*%lbrE^jh{(h`OGn|5J5pB)}}%CFNGJKLE>JA z`Kqbh5@#a<=etvp`xPy5arENV@B5xa;L|EB)H}@A+K>VV^ZVNiwH&D|R@Ic_%LS*Y z8f(pmllY*OE96ZdPhgky!wI2^sw%alub`&H2_y*~_Jb4FBg>287D%9(wR%Trb&R*! zgIvuUFdNo3IHmHxmf3pG_hR{ce*bHeGbyL=(Yvi~^^vr;cG)ezvr*x*wHTgiucPI( z)rLh^+&LpR{8GL0wIR;RjRENgmyMc>{W$^aF}gtH0s^hdWnE#}cmKu1WkZL{3{;;l z@XlMW(nQ_Sk=v>5LeC;F$Nv-u5a;*W+Al#5XF2#JL2^IPT|_>pU~Q_OeXpMOp@05S z{fwV7#^!{=fvd2LA6>@K684dH|1@nM{-@4v;N_fA4@Kd>^H0?A-pRR_X0bw`1fz9yNcs8Sh|k6e%hdzXdi11_p}%^=f1+o#Xq>JzlJ%z_FLItlpS9kdpz;7=BAihTax-NEgbigrNax$kuGV#f%I1RcdNMP z-$B+?qwXn9X?kiQ-}4V9fj5>YCjKY#8lszD#KxsF@x;6iM;`8x8IaNfZ@m_85vm|U zrm(&;0_Nx5oJ!{LOQ(TNw&9=sKqbM<3(F4_77>89%iKZ6qo-u13&$17cuF zp7xmeE{F9@J-s)9S#D>cfrmvQ_5&<|R`*Y~@)FhG5ZQh~o}8cu(E)ldqr#pKdL#}y z#D<0N9=r6}=(PXuWZ>VGFZJlmU;%lHNLX1E&80rE;qC)@cnz`J~bv;yQ*7^I>@}J=1iay?*iC+FT-<_*u zl)m|?Sq!{9)3F#}2M>kn8A2%bOU=nK+|AD&eiXj_LlO=j!}4UiRm=1n<3Tl1nLGW& zS6Hu`jLXxSfXm$&%+qoA8PMIDpvS<&rtNIoq8Lr0capy6#WVDiU1n8!Cxved_n0&= zqIEBZi{JTxtAWHNYOsVldYH;h#AYQsPWpNU=O@JjofANAM(~F%o2a*f4YAF(YJuqrK58h}JNQ$Vri7yvI3 zO(uG<)ROv7(tr=w5ZldvmP*GYpig9twy#^qDvt{KXom3F$pE8b;TiH?S5FU+8ZcnS z0D~VF|MKm{erdogL?@x16A1AVKaclG@ci(L=1Xm!;~NRtsU=N!E{(@vJGGu*O5aYz zUvKgRDL*G$TXY-W48dYx;^d5JZIu8PxqGE6j9SRvz@qtQ5}VuI$$)_UZ(hL6`@+xK zcMCI~KYwoayXsvrhjFv}>ZmZnzUuc>S2NPFB3rD-2V!F|zRJ6MKhkHe_N1)kD8hOv zAOoI0sk0Ny7s=|EYveZbCHs|itSFaZ=!c5G6W{vNe5=9FfGCb*_`h-QzrX0axbWTF z+-z)by8+%|5(q1&XJ@;=zJowm1op?9qgmpqXJg`(dQ-8q!f}vMWKO{4eA*Z~ZRo$) z>Ef|B9R(;>wd4Sbu}Xu5%d@kK|ntQc&~p>?KuxmCCQY)xP;u}!~MUG!NWF6 zvm-N9Q|_N==c>*!#2dzv0&g}`ThAv|B`&6Qj9h=RwqE|RGxC^$qWqn&iS#+k%(kcm zjL@;=KnQTRWbHRwv}008d#T4eHwNDw=7-}*X>sUriKOq3a4I~8DMGWI}>4i z=KWR9tFIym8OJ>4j0ccM9h$&2nzqvIy#M|fXuOzx%W`7BQ2)vAs4dlUIAx+ez^^}^ zas1bJx&3wrk}6I>n98yyCJew*pk#WlMUjz_L8MxMnX*)Fk9_TU(XjX`Fi;9{f9N&e z;Kazv${I^85V7FYk)_Ds6E5j@xpF4Dz0H>8PSWOk(Y=0>kdvb+D~oFCe}Vy+^!X8j zF&MC^l$*2df!;G97VtSPiZr|Lz5=vc4G+Js0suMc`jTUS?z?2{|qf{#zkHiKwOgMfPXQRmI{_L6}-vTAcvGLGfPwNGWnq`}<*9 zA&%}RVAJf}+=bQEpY=@x>bmCo{%Mflw|_TTpqlFX&rY0IWEg_(Tu>MSuis%3GK}D` zWy%0Z1Bm__lRg_88^C#fBQI9Tl#RiFksVJfB?o1UW?%w?LWYB7WeCmUVCVWXW;E*p zy9vUxO|IBq{^V(>mtXA7#B_JR@;qGpx~rA`F*B3S_hJvay*rfxq!R4_AK`L-7Lp3B zqx%g|P{B#TeKt?V^J66Url-RYM{GPiJX7ze^q=i9IT9z@1s8PE!*5^wA~hg+sqtOB z>^{xJc5n-3?@pIHSQ zlCtKPf8r9BEw)XBV?tk|OMKLcMIR%VJ1Tzdhx(RQKnQR(KWJV8ImlIU0|M0jBJ+X3 z=oxq({27Z5*MBkX2apLxCD)5#6Z_y$VYyQzzFILc%Y<#LlQEK`NTX=Y)PYZ*98(ja zIM!3B0XW@omNE&*R(St&Nb~)H{tm$8%&LI0nUhri|MR7A_X(!i*!XIRK;~nYOd$(@ zp}g_}i{ht}1_1BJGlUIkKYxRZoq1EErf8`a^Lm{;T%>{|F{kMW2^&;qR}zc6H~s`zx`f2rJ_*Ifc4x&N>yDZuKhAk zght&y_guHovS^#fawem{W(S+dphEI-iEM){5h!;r_X4sr!Y@%D^Wcvff{8FS%x53~ zbQGa>^M1bd%mo?#aNX)DuKekLx`E1pPfK9uuVN`2R34lqwuvH@hyW$)(@lqC%PaVlyz~N&mZuaV*$M@gu z^`h#PS;X300$SFLMMd0h>-?5(6!nb-fC$o3z{!_ivm3z$Yh*{p4f-d z`)5E9`?2TktAEKoS-!TiBoxApuF+Kz=B*7yDEV0cCdki`tgIGIy2G7v{xRV#>0RF+|65+RrW7gqEZ75*j1{E zGk_Uz3^q@+ZTbW;ykA2Y`bzSb@r;sYi-DZN!ib|hO$d!GMu}ENx9;ivJD};$TAG+N z2*BD*EfJsyKrEVInaK~o{x#O-@E{T{-jiD4`o`%jdAwh%Ua_DmRf5~nEbXWBU~=1LFp z-IV_zE$;e}OSY3kta1xYC6n&UeE2ZbBOcjq0M#$xcO(7^5SS1P_35fOF#lUfyZa;K zDgh<24+J6v8{{CS)|*S_A03Rm>QRjEN-3z?;JZE!QB$-~i%C3mY{~g&@GZFBrqFnd zWiJFM6GXTFppMC)>Sn*?TxBDrn<7>TDyXtCv0CRV#89>i)tb5KJZ$XR28}mW=76|i zG!Ly}DJb-id_S>ufyj{7A1lKiCP-K2quP;vb2VIThj$e)?_elgtPwSJULD$iTD&Jj zyt_wm6y=ucY4Ba#{;iQ(l1l~ezjy$OuuRWCxk6bp65YtcTt6CfhxFG{%{;?~;ztb@ zfhQHA9m;N8QR!p0p=sIZ+2~YK{@%&s>H_N*U0uT@2I8k7x4f8K?+g6j#SuyS&fDe)^_{GvF3Pz+6A z46GF|7X30JCimaMwCjrPXRv`4Q3v3Abi_6o-a22KV6Et}Ah8%FW@jFa_uldlNAz}7 zZcoKW%CS*z+TgI?Z4`hizAlk)btlm%a8~@rsL^+_-bwvZy%e6xc2hi53x{`(z8swb zX0WbQ@O6K*lgf-S7#tpOsuKkRkX$nb#vamr;xNj7!?AZzZ=s?j+eshwawEIS)=qlw-&&(&w znmJ)PjgRAYu`;pBQ5T~YYI(}(Ou1sy$mWx7zCz5Ow^r3yxYScCaL)G}UF>g2-p2r> zU3%100=;xph7&g0Ol70_I}1R76c~6YW&)~939bdI z3?YS4u#S+FAnIf7Y2;goMKWxJERp4N?@L2SO7!4lmblk5nm*u{pyG`3(*;;{XZ{aCtCJ z=l<)kqNqVzVnd8rEXUGuel=Xn@sGTPUAkTvE&|8Ru*Kl1x(GCuV53ZdJ29P^Fqxvp zclvYI+5ISEZp_iMVn4gGao?%6>0UvaUUVuEi_&j@B|>X0haU;+IrmSeOHy{HVJ4}V zsxhK43clj%pU(Nu8J*Pje_T>6N(z(OnZ>9v0I+nM|L|@doB#VP+!D$=n)K4^W#KZH zY}yvPQ`yuAd0A5_pNECKTA@$%Rz?4G27uZn4Bhn2_eTGRmJ%e4O0RpP1DxxNPE$Ke zC^qD@RR{XRQY9+%QJO8ANpGgnvDWLgNRNb!z9}y$1*}2;-=#nQbxHr0O971Y zpLYL?H-`*}lVD^htTIH{#e2@gyV!T+JGf*hslZ<0*XRjBP*?C2y62RC(WLL4q=w5E zzCwks4+XLMGsdrvj?k9%XG~)TjSrFH0;F$j(YlDHtShZ2`rj*D?e`v#-mahh`DmE* zZYhvf$NH-V(a9KAKj7Cln#!O5-?adst;t@}f0*VUX=|Cx44{Qb6dg8#L3e{^BhB|h zg=73a@X}5{aJ3rbSxZlI$(|6(RnXjZ6vUz$#zmf30MO0$z0Ra>0Z0S@7!;rj3P-Ln zXUH=JgihyrHkOb7ud8VwSd6@TnuPN{7ES;fLv^G`{86Ho57&h6IYa4}fkfE zvvSl~clZH_2*VKny45q^G%*(xr)7x<0DBo=c<3YvQ}p`n+;Cq%$k15k=Ha*G&aXmO`#PyV zF(%s7w@M9mpiw*$hgt>)duJk9%7FN9Yv=>BdFV%gmg2ogKu;F*-|{8#u~(tLOP8)B z6x_HASt5L=77ZwA%J(DoNRHAzyDK@WBs4(ZHi@F{onlK;SYdDeZ4LNY(^KBPW1dzd`ucp@GI@;QA3h_kvGIV#e=y?-a~Gxt6q zW6b)^J|KEZ@~DF1J$9_$2reM)*opA?0xo?4^lRk~EW6bA4oT3O^S}1q$Oh3M`J8vj zvxNmU=&w~*Gd71f2~h#|I$^g;&0*t{$D0=zDp?uZf(9O$G9_-wKWr+D)zc`3 zFR$OIxYsOB;|*BYvNt!`A#}1FMe4Z=nHIyLQ#uOdJ1neh_`^WYW1Y2RjQaPY@seeD{-)S!;_78WF>uBTo24 z+NF^54m}&G*=XbWEKiez)XxK@Q4X3Kqx9D7D=fTW3=bOc9K`psS#8$RK;G|VZsoIX zc}?6x(xj&XUxNjQ#05XMNA;`)dYI*HSC$2s~moV+!OjBWl z+cM+*aV%5(hNyvOZA4Wd@aC?Nb90KU=`?oVwxmh%LNi+|M*nvjc zRYtQZxk5wi*`^pmd@oZ5qQO0_Ot*x|&>V`gBUFs4U-r(jmggDqF75~xCa3{4hl&g? z5YeYb=m`h3A>Y#oR~$~wCUMw5Trwe5IBin*i>`>vX%1wq5r3ryFWW4`bx&Z<7D&2V zcl#L;Txq(1waXBqODC3o%tey!B7484&^ul@ zM-%movtL<>8h6@Z_ks?5yNMNNTn!O2^9+N#DD<}?Jq?v&wUy!93*Uw9P4UUGm`lm& zM0;<_>bJ(q?*%J#k?)6o+kAr6qMCplp)&#CQ>tUtOn0Q}N<$|;PGQD>6D;c(i=B|c z-?B}>-n(mFQc{URWFAV??MaLD@t4ox-ODe=Q1iI4P?ORjKh4JHB0tOYp~?`@pATO$ zs1TrZMHl;g%sfhe^AsCrT--68CB8CU1o91u<6$(>pPa~vO?(1+L|=x^?BYlr;oj*$ zvGj!*_eWabo6zME(V?Scu9_bu< zwIy@4p11euP-B0Oo}2%NVjOA(re4gTir4R@&Q0*=_@qhPU`7jIXTX+XDW7j*-imic z_stO^#ryVt;oZbh)_dYtV}^%AsVf~R-FH~4Q>j@e8jW}wkBF^Ak#4NJPi|SanPGh^ z!x6;7-PBil#smLmA)crC1+O2WJ_5?kCA>AsnL7R3gY{n8iD01_bsiK(16z9gD^$Wm z;WRgxbx>#NH+Ya~}6}kWd9l4E-^gMuVV2$Uol3!)ewW;;PolO41H9zyMIX8wkh+wh4U-U!yb&cCfJok~twDQUW zIMYut??Y!>81mgV_u;JZA_af?&28*;&0Q#lR;}2HVbCOl`^7Iw(1iAU|HN90CLew` z2?UfTb`VrB|JI9^e{jBbEvcjtNg?Z@ajajdvf)Mr5k33qJ731 zTN|IR5_XWBf80xbF^J`|#tHvAFFP7yGD@PdsfSKIrlMs3?hWjpxZYaMd6NirXpY8? z;kn{@lW}du2a{YGNa)3S)NS~o%&AJ`K@=3x=G;@kabO^E=)Eg(rZI=_->!b(+W%*);An@NZL6jJ`(2PGAi^&)*hm>83jIn_jI2lx7jxs5R7A3=2pi~ zj*rB@ubIEtd+af_8uG3c_F@IKfIpCM$xh4OQD1hlEF>O~4uK*kd#x9){>9JQY@fLc z$G_>(31tI+&av5+OWO6DOqRPDU1O3Yq;|w9sux7EP_iGSkGm=}XEjdbb$;qHtmGf? zxaNAFs3xE`e+^q$ATqXKJ z%Rcg&QT6>vMDOjWB$^BgX7A z*2>D?Xq_MB{7wYh;!}KSk}_HOYhgwXvj3ZjHYx%4jqzOzgP-B8vEM70T?o zGJ4{79NkL?OSR|P61WE& zVQjQ$mC3^fi5}S@hS|-QBpz~xu7Y++m_HsOucR_ln%h=D{srp`+g4ED=T9v}@l}p~ zI0vqG?slm?>;s+^pU-XG5}cGh=tIn4T1qyl>(-Vl-f?s4-cik=N_sf^;~~m!1Z{+v zpG}NKK3TPS<7l(b*NW}hUi&gPZ;}>uDiK}?zP4;q9=;=;1N9z-)y)&8_e9CM8AR$D zgcgUH8BMK>wh)qFN5C^DcwjA)fRIVf9?lL}6E{$Zu-R);D7Cd2V6j-o@~@Rgx@rnR z`+Mbn=4;{x#1=q_MB)hvIrEEHRhs>16|lV3ld{EX9{!HWA=hY?A;UIHp2`=T7V#NM zdXCFIcEeA0XSr!}2IQE?y0=dtpN+1Ws-1hdB|7)Yk=Wsa)g#Wvistv%-NS~}@33@g ztoXENS8Tqe$e!#IJz*c`=Kfa3-z)bJL-nilMV`tsh{^lTQmwKz;bV}2!P%{Xsj-BQ zn7uK@jnW&ho{S(2vOC(KX;jJcu6wc@ltV%7Oe^Ae4YbZl6&qQh$L zmpL2JOo5EC(l;gC^F{l;OcQ~mCMI&M6~`hqDdg2i8H|twq5ULUz}$Ja(#Fc>tA$;i zA1}=;^V?VJaqeHw)&SHcLnkMZ?)IZsbKPo<1M)dx@mmI?I{Om!~`JaWt-VekCV$rq(Qjt}}H zBkj$$iKe_VJ7K0j95(%0y{e=&y}9%J286=D$m#op16&{bn-gw_%S&^QmG`rb+*^;9 z9YaRr{)3@&az*^*?@3s&9$<6&_Nyx$!;N2U7CG$6Fo1`uJ+Lx`0?XZd&h@ewf*<#PA8^yGtSkdd3W34%mB}Xrs?~u5EZ?@Hz}GR zKZDmh{=${L#!>w-Qo)W(I&ThsYP$DC?10CQ+#TfJE!M6Eh)+PVMX;{-DvEo47~lQQ z9ZWvk>z4eWL>&KAO<`i+)zw51#l-YSL4R1FyDP5U7{2IVg(fi|{fK?I!1@alBigz; z?9333fA>dxA1t1rm>E{gaV^pPspxz&NgbhZn5vYono-7~PxMYSkPxU*kqcqla)^EU zys{IgO^islkF#62g|KR0rr?ZP*72%s=)?G%A-}BdpTZjPne+pXN)WRX{g!{!^rOcu zU*_e?VmKDEnS^=QcOX`A#&#Ycw?h=`W7t3Arl#~>6}nl_GETL$_bD0nD2k=cd_=e= zx5{yrFu?%z1${@^i(}O7;*8vKm8tO&@)n9m=iZoaccZF9yjZhA#0)uY^AJ8j{Q8;SBXx-vHhWphgOSlghi-9dNqcJ3@_sL6ww1z; zwm-?*K(;>B`h9G5wz$V{p#3iFK_tKC{F% zg#Ezh37e_(Xj6#8rs<;Ji%buEj=$5F_TL303#d8b|tJ`VCThoj=jnqKa_LnQ)F~-_2tp@OgQ%mkBIjlxxUr%pFlgy*z?lr z%6LD23ssrScWXJC8SE;u-?yTWei7T~f+O|^Cf+_r$2{~Hg@F(q z{&=t7vJCBC*ZK^5Dq=XC>kxLW%Y%8PnDfE&i4pYD`m~`nOp$|Xh~{*#AX%i%!xtxH z!S{KlS!-^%?lCO)NqIpgZR-)Qw$yg`0c{kW)TPtFkwy|HdF)9=)Kl1?p~dX?S(#Or z;4~GR_)enx&y0RnCtAna3VyQ+YcbhJ2g~{dta0Ut<2|Qg?~;qUNP=8+>LbqeU^W?E zvNb(~ozZqcs-8A+iPQ6?>aX3AsZ%xnAQCUaQ2iC9c~#-1{b}{*M&E;q@?ZIUsHrir ztN+WI4{h+!VxD8OxXCOX-FBktdMO|s^*7S4;b5~CYrEP!$VUa?*Esm{ZUKkm?{oHe zQJffts~O9WT(KPOZ-4j@KGN4!5+;lmt85@~bO=W0R1->+NutMDyf=PD{T>?i+c}-6 zY!W=DD3|zgMoBEi{hOXtny%S4PO8MF#+zny*=sQQ2}eUkf8I)#v(qT;bGdZWa+N%> zbEX6>rLUJ`V)cjYefus1k|r*mf=t@8(m(?2dN*b@eu*RJ*HC1)VBcInYBgEnv*>c^ z^^a1Si{GWC=HZec%$e(vXS9s7YWW7- zB zrHE zEy~zCPuuF6{LJKxtK;6*Pp$3mai8FjI(Wbr8pP_y-VbM2r-^mnt-!<22*bbdL|?LT z(U|Cyy&b1&bGog|%uPNjtZn}akcl3(7E_L)hYv(}U9_AH)sFA5uJR+;s1w=ytOX={iU3jA}P|M()nAH6rb@mN$HML(i>7OvX!JCwnt2<_cwo1qB`vga^O&p~v|y(e{l= z+YmqD;7ZHC^M;u9c0#%=b>fEy1%h^OTDG;_9~X-ww3uTvgec}!7!^~-&*Uf5sly9h zE_B7t1cNc5AOP}2#>8DeIT2Yh-q`rG{(I_hE$ORQLdS)r>r!sE$Dc_5c`6$RvfiE{ zEoq5x11h?a^=wtD*&0w+0{`uf2D;qP$NheRx_w85>hxs1)u`^8%))WF(Y{~&>=$wC z%k;i}A-2gn{tW4TDBgiDI$KQy&?+qYNZ@9N`->xk`h5N9h8BY|-NeAcwL?v+-WS`H zn&nka?9sI|Ek;bg4wuiYCS>xtB&P2Bg(?@lPo*joj-PaICvxE9{o`TxifcB#*A7z$ zJRL?;#R+uTJzX^fzZi8yRu5@)_dXS2V_D^#{q{=Pkab=v>SlSJ)EivnEv@Rkxpjd} zne#hFKb5>cHpl*9Uh2C;ef(s!ncB!|bL$Q#jN|NAmywafm#@HsOK}=w`&2i4-b;-7 z8>b@Zl42Wc7rx-stQ-f#z>)Ot&gm#nW=G=AzssjQ~t zil5~gV=vZ7{hF@i7ki{roxUR;3b^nHtQ3yH8qVXBpO+Mgx6XXMEr*^5A`dBB>)+$D zS|;maCd(+@*k??U2@TxyDVi>YbM9M@cyjDxSYA*R^?e>-iIr$(@yd2HvJf?iXDjq^ zUZ4xg|G+#SU#%Z$d=)>uY1j~ zZ?8TK*SA?hx29&6DosDVRmiYsOvO7IMbbV?JdzH*T%h0+=1rjPiXq_!WtJlDmlSoH z4>HmMN}(yK|FSxu6qpo`J%0R=gMakAv(H9^M!r+zSCiep8Ro5a26QwdUCh!NH0C;HfU==|*V-s!Dz2)m1y8=m zG2749wtB&-_NO|p*(xFGjn-2;Nj^ItX7MXVUz(J)r$us8zTV$wUivp9?*>IyEl8x& z^yhB70kE*$G~n{@-Ni^zJ=vE2ycCaEce9_*2ZvPcymdUtU3SA>fJmF7kK9t84Gwq{ z!|DiAqr=xz2YnqJm1iw}-BZ7l{eV%)>6JjuTRD){9Xt38*)3I4v zrCJ?ha*hp+nY<;YUZvIyIwWr)I^xYjIV&udl6u)zai2+;m?tgm9NqdpLMj+vV#F&_ zk<9ov13AQM{qA3fK60G_V21#VbGsLqMa8!&{q^#YQbOvHvlV?Fz$)&9SUlk8W3n#> zj#m4L1I#bmY>vuUjFb}(&Ul+mtsOIdF?P)9Y^q%6lKRQNzw-U2W4HZ{A&zMbnDU=%r|`${sSNO@T(^QI=Q5HU8pNr;K!hL}F@fV<^JcqHoe{CW&d z6B4e^G`t-?+eIIfDVK{4>POZ=V6UjTP8J7C~HIa5!f!&-s3cwPKNdy6AEzz+?la19bUB>2>d68v% z^---xPtC)9@v+_5d^q-lNxbtEd%%gWL}Hk_d(0>2n1rgkW%{(2mbz*+-PZzex(*#9 za>jwt^;_soh+@w7$hX1yz8$je3rA}pa&$SBdQGb7rJKZFO+h3xB1+mv^jyPoCtAZ|OT4(R^I zaFAtH?$Q_bv)-lJfjUU|zRWV?=F*H69E#xNL+s+T{W<$Xs3>rd<p&KmVK1lOkz}a{eJ_lu{na zPWZgZ(4>N4o~;@`H{+Wg+uYH2j}+m;KHUmL&st0W*>$VBMnD*cGYGLlnH3%pNItAmZil;It=0~( zRa>{ds}!NTeRAK&9$)!GYas8FI6tIJ1Su`hNh2h`i>u;#)>!RJRr{N8Q!gh4l`oHt z-IIH-lL(7M?ZOiS0Qm5jM*CR**Ms28gmnb4RW)!HgZrFNL_XA^I)G2#e?mX&C*JOI z`T>#yzEpT3R(I)E1y%x_9JnekKX)sNCT$!2YyQCZ4I! z_Ei?rMxNy0y~J!G1gm$a?terS{-&?7;pY-fD=cHbOt;6$O<{25yahJ@nSs@DDwHeE z{kim<4NNKz)Md^yfxRrnvb`%`@Ra!g1yv;lH9-wbQ_Q*(`6S^m?+|H}Z4{saQ{KYN zD$f-?lsWKXypE4N7N~gABJtq-Y@^O0i^7IVt9!u-Ek`lI9IJenuJCU^>!J4Bx8*bC za+Z*-(YfhMv9;`^U*UD9!i)XGpEC3-5OVe7^F*OjMxH~z^H_Rlv>O{j7~>?xCd*Zx z2^IwWumid4O#MoKLl4n3bDKuKGP5|x2J=&={cT_30hju5?lGQ9qww>R`;6 z6~Mf8(yeJe@EWW_c1}XQ;TEr;GctW4Lu=g^4;2BCuz`I|k-Tt$qOB1c$2c}Hv|rzY zW>>GqL6EA9EPf!>opw$r8{)$HAaUbW3ZLkytG<)jJOq9l*A_=^s>g}{5iLA4`5TO< z#pAD-V&BeRxi=m;Wg^ep%Zwlr#Sx(eNY8IfhNAR!x%OVNKgV7;p)A)M`}$V9{y~HE zYc{v%w>k*vr!ZdUd!LCpwQx(eH61jIxdcaqk^|j-ADSeImo9S1MHs_tfq{#w%X*Wh z3X@>o(3eWQTBxszb8FF2og7;nX9ADlG+ayiWQz@MxUlP%b9(SH%_V{ZY4$unWN0#c zK~ok7qhtApP4Z%13Bm{}tysXXt)og5)t;Z`+4%3lIE(dgon#o7dV!CDj(cOjql++i zOpD9Q&Blyh7!)f;eCKh#2;AT4-wgm2lee?QStDIBApQwF!MVb}azq{Xlgj)}{xfKA zJ{*;}3B#LmyM$K&>fsR46P@~FtsK+&e|wy7B;2nM0!kk8HFY@O|0}D582oopKJ3kh zp`~D$k`wWb7DS3N5vYPC^|}rKSIa~6nuE3#)X#KGbGculpkPJ*`(FSr<94FN8UZT5 z5LlV#`rAE%{pa8s`3=>pN8&?g;zY&8M*XP4D!jhCtYU{|ZOCQRV7a=7U(BEy@jNrw zE;nMvmo{dw?(o_)da#Z<8YxR`xZQsl0dsebydqDWEOvPGo)%=OzmdWD7aBr`M`vGO-7Zr+nBqyPk`rU+JG^;?ym33c#Od5XWXff6H3^w z$PkwML5Y&J9-$68bTvg_-1P?pa32eeNeZxm7+;1s9AVV2FZxeEZ4l~wB_pMX8o>uHHl zpi70oW0-TSo!cT_g=}t8$JOwZ(!FP9pfNEa^8Z7D+%vRTSInx{(2feL%veV!XcpmL z4I&+f`V*k=6S`CmN1)Y&M17)ffVUUHQLhe2GLZ$JUCo_};4w3zVCnJi9_leq7sS6> zAQk%KMN^oxQ;TU6_6RzuS-=4y-#Zl#zr9@?KKZ)(9zA{b_W5x(jEl60lRVfF-i)8Q zf#vQIDO}Ll!woWt_C-uLco{e}xab%~(mGdQcb`HmKEg&w6Lpr}NxHnpAGTPmn-}J9 zZ+jt|U(xRZle1Kx7RKtgz{q zyN)u8deoYroYKAX;?;@GGoV(yoa29M#bb05=ZYcub+B6ej{cHhqxF7DM^?8N$HG?^ zh!DHQ6d^W{U(f#)kOXi0#UvSWIZ?$+O?`XEsqHp?7dT5qj`;uW-A{;BWf;ft?+=R7 zgn>$E3f4j>S_EM!8KD$u7%jq8f1osLkP;oS z%+wYI649b8wQPD?yw7zm*K_a8mAvDOzYknyc+Yd@Jm<`r`=0llJI|Z9M`mVQM@hOH zcm|ji5l8FV$EwUJgQN+RV|*xcKO*VI{5x4d|GsuI?{?3Pp?}ZmBJ5LCn#5@51-C4rrh&CkgtV9l=lJ3 zz4_XCzBXg;%X4TquoJkq?&r*)ydfV1R+CjEI-Gag=f+UK=eX2{nVD7cEhaNFtK<&C z%*-megD^9*O70-c%&d|-2s1OQc+wgT(O*v}ThGP*i9YE4J;2od~SlMEo@xZ){J!2wb(ICQR z16zP^D%M|nYPebFc?4m@UX&6Yk5CGsDMt$1B<;-C%^CUva9fT%x?9AZlrj4xZO9^P z?@1b+%kId+aC@@&+6CEu@@ihvACgw(D5po#Wx1VMz)P9S#D;A?C+VrWpY@`o9g_Cf z3eZU!C23RU^e!-}ZvB~(zRZ2vENP6SE|qx&CS}YtIeJV3^wk^h}fRrzfCD?G70!0BKBwd z2FfHcjsB+GN&@(ra+|AN?^mpUd*(t!^is+LpRN~woefx+Ac zvvZuELD-0p#o78Oqk@&d&%iXwMbPmkt91`DuCX3Xvr^1ePDFn_n?6qOZ|<)q>sQsu z5oJ=Jxu+~B5Am-Bj!|ZX`>8R;te!^@Htb0n3#?9Te zPe#NxN(ZyzG`4H!5La_GYe8lD)-4e+Wr$NT>o|`f%*?EkM-XOaR>>m>Gc&8?4#LdL rD!GF&GqXzWAk56Hk~;`9GpqC;)!6A1LH0Jx00000NkvXXu0mjf9nuxh diff --git a/components/outlier-detection/mahalanobis/outlier_mahalanobis.ipynb b/components/outlier-detection/mahalanobis/outlier_mahalanobis.ipynb deleted file mode 100644 index da3aecb912..0000000000 --- a/components/outlier-detection/mahalanobis/outlier_mahalanobis.ipynb +++ /dev/null @@ -1,577 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Mahalanobis outlier detector deployment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap a Mahalanobis anomaly detection model for use as a prediction microservice in seldon-core and deploy on seldon-core running on minikube or a Kubernetes cluster using GCP." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- [helm](https://github.com/helm/helm)\n", - "- [minikube](https://github.com/kubernetes/minikube)\n", - "- [s2i](https://github.com/openshift/source-to-image) >= 1.1.13\n", - "\n", - "python packages:\n", - "- scikit-learn: pip install scikit-learn --> 0.20.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Task" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier detector needs to detect computer network intrusions using TCP dump data for a local-area network (LAN) simulating a typical U.S. Air Force LAN. A connection is a sequence of TCP packets starting and ending at some well defined times, between which data flows to and from a source IP address to a target IP address under some well defined protocol. Each connection is labeled as either normal, or as an attack. \n", - "\n", - "There are 4 types of attacks in the dataset:\n", - "- DOS: denial-of-service, e.g. syn flood;\n", - "- R2L: unauthorized access from a remote machine, e.g. guessing password;\n", - "- U2R: unauthorized access to local superuser (root) privileges;\n", - "- probing: surveillance and other probing, e.g., port scanning.\n", - " \n", - "The dataset contains about 5 million connection records.\n", - "\n", - "There are 3 types of features:\n", - "- basic features of individual connections, e.g. duration of connection\n", - "- content features within a connection, e.g. number of failed log in attempts\n", - "- traffic features within a 2 second window, e.g. number of connections to the same host as the current connection\n", - "\n", - "The outlier detector is only using the continuous (18 out of 41) features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test using Kubernetes cluster on GCP or Minikube" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run the outlier detector as a model or a transformer. If you want to run the anomaly detector as a transformer, change the SERVICE_TYPE variable from MODEL to TRANSFORMER [here](./.s2i/environment), set MODEL = False and change ```OutlierMahalanobis.py``` to:\n", - "\n", - "```python\n", - "from CoreMahalanobis import CoreMahalanobis\n", - "\n", - "class OutlierMahalanobis(CoreMahalanobis):\n", - " \"\"\" Outlier detection using the Mahalanobis distance.\n", - " \n", - " Parameters\n", - " ----------\n", - " threshold (float) : Mahalanobis distance threshold used to classify outliers\n", - " n_components (int) : number of principal components used\n", - " n_stdev (float) : stdev used for feature-wise clipping of observations\n", - " start_clip (int) : number of observations before clipping is applied\n", - " max_n (int) : algorithm behaves as if it has seen at most max_n points\n", - " \"\"\"\n", - " def __init__(self,threshold=25,n_components=3,n_stdev=3,start_clip=50,max_n=-1):\n", - " \n", - " super().__init__(threshold=threshold,n_components=n_components,n_stdev=n_stdev,\n", - " start_clip=start_clip,max_n=max_n)\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pick Kubernetes cluster on GCP or Minikube." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MINIKUBE = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube start --memory 4096 \n", - "else:\n", - " !gcloud container clusters get-credentials standard-cluster-1 --zone europe-west1-b --project seldon-demos" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a cluster-wide cluster-admin role assigned to a service account named “default” in the namespace “kube-system”." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin \\\n", - "--serviceaccount=kube-system:default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add current context details to the configuration file in the seldon namespace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create tiller service account and give it a cluster-wide cluster-admin role." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "!kubectl -n kube-system create sa tiller\n", - "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n", - "!helm init --service-account tiller" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status and deploy seldon/spartakus helm charts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/tiller-deploy -n kube-system" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usage_metrics.enabled=true --namespace seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status for seldon core." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install Ambassador API gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!helm install stable/ambassador --name ambassador --set crds.keep=false" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deployment.apps/ambassador" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If Minikube used: create docker image for outlier detector inside Minikube using s2i. Besides the transformer image and the demo specific model image, the general model image for the Mahalanobis outlier detector is also available from Docker Hub as ***seldonio/outlier-mahalanobis-model:0.1***." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if MINIKUBE & MODEL:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-mahalanobis-model-demo:0.1\n", - "elif MINIKUBE:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-mahalanobis-transformer:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install outlier detector helm charts either as a model or transformer and set *threshold*, *n_components*, *n_stdev* and *start_clip* hyperparameter values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " !helm install ../../../helm-charts/seldon-od-model \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set model.type=mahalanobis \\\n", - " --set model.mahalanobis.image.name=seldonio/outlier-mahalanobis-model-demo:0.1 \\\n", - " --set model.mahalanobis.threshold=25 \\\n", - " --set model.mahalanobis.n_components=3 \\\n", - " --set model.mahalanobis.n_stdev=3 \\\n", - " --set model.mahalanobis.start_clip=50 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set replicas=1\n", - "else:\n", - " !helm install ../../../helm-charts/seldon-od-transformer \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set outlierDetection.enabled=true \\\n", - " --set outlierDetection.name=outlier-mahalanobis \\\n", - " --set outlierDetection.type=mahalanobis \\\n", - " --set outlierDetection.mahalanobis.image.name=seldonio/outlier-mahalanobis-transformer:0.1 \\\n", - " --set outlierDetection.mahalanobis.threshold=25 \\\n", - " --set outlierDetection.mahalanobis.n_components=3 \\\n", - " --set outlierDetection.mahalanobis.n_stdev=3 \\\n", - " --set outlierDetection.mahalanobis.start_clip=50 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set model.image.name=seldonio/mock_classifier:1.0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Ambassador\n", - "\n", - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import rest requests, load data and test requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from utils import get_payload, rest_request_ambassador, send_feedback_rest, get_kdd_data, generate_batch\n", - "\n", - "data = get_kdd_data(percent10=True) # load dataset\n", - "print(data.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate a random batch from the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "samples = 1\n", - "fraction_outlier = 0.\n", - "X, labels = generate_batch(data,samples,fraction_outlier)\n", - "print(X.shape)\n", - "print(labels.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test the rest requests with the generated data. It is important that the order of requests is respected. First we make predictions, then we get the \"true\" labels back using the feedback request. If we do not respect the order and eg keep making predictions without getting the feedback for each prediction, there will be a mismatch between the predicted and \"true\" labels. This will result in errors in the produced metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "request = get_payload(X)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the outlier detector is used as a transformer, the output of the anomaly detection is added as part of the metadata. If it is used as a model, we send model feedback to retrieve custom performance metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analytics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install the helm charts for prometheus and the grafana dashboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", - " --set grafana_prom_admin_password=password \\\n", - " --set persistence.enabled=false \\\n", - " --namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Grafana dashboard" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n", - "\n", - "Username : admin\n", - "\n", - "password : password (as set when starting seldon-core-analytics above)\n", - "\n", - "Import the outlier-detector-md dashboard from ../../../helm-charts/seldon-core-analytics/files/grafana/configs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run simulation\n", - "\n", - "- Sample random network intrusion data with a certain outlier probability.\n", - "- Get payload for the observation.\n", - "- Make a prediction.\n", - "- Send the \"true\" label with the feedback if the detector is run as a model.\n", - "\n", - "It is important that the prediction-feedback order is maintained. Otherwise there will be a mismatch between the predicted and \"true\" labels.\n", - "\n", - "View the progress on the grafana \"Outlier Detection\" dashboard. Most metrics need the outlier detector to be run as a model since they need model feedback." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "import time\n", - "n_requests = 100\n", - "samples = 1\n", - "for i in range(n_requests):\n", - " fraction_outlier = .1\n", - " X, labels = generate_batch(data,samples,fraction_outlier)\n", - " request = get_payload(X)\n", - " response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")\n", - " if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube delete" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/components/outlier-detection/mahalanobis/requirements.txt b/components/outlier-detection/mahalanobis/requirements.txt deleted file mode 100644 index daeb42f09c..0000000000 --- a/components/outlier-detection/mahalanobis/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -numpy==1.15.4 -pandas==0.23.4 -requests>=2.20.0 -scikit-learn==0.20.1 -scipy==0.19.1 \ No newline at end of file diff --git a/components/outlier-detection/mahalanobis/utils.py b/components/outlier-detection/mahalanobis/utils.py deleted file mode 100644 index 569dd54ba9..0000000000 --- a/components/outlier-detection/mahalanobis/utils.py +++ /dev/null @@ -1,171 +0,0 @@ -import collections -import json -import numpy as np -import pandas as pd -import requests -from sklearn.datasets import fetch_kddcup99 -from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score, fbeta_score - -pd.options.mode.chained_assignment = None # default='warn' - -def get_kdd_data(target=['dos','r2l','u2r','probe'], - keep_cols=['srv_count','serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate', - 'same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count', - 'dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate', - 'dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target'], - percent10=False): - """ Load KDD Cup 1999 data and return in dataframe. """ - - data_raw = fetch_kddcup99(subset=None, data_home=None, percent10=percent10) - - # specify columns - cols=['duration','protocol_type','service','flag','src_bytes','dst_bytes','land','wrong_fragment','urgent','hot', - 'num_failed_logins','logged_in','num_compromised','root_shell','su_attempted','num_root','num_file_creations', - 'num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','count','srv_count', - 'serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate', - 'srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate','dst_host_diff_srv_rate', - 'dst_host_same_src_port_rate','dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate'] - - # create dataframe - data = pd.DataFrame(data=data_raw['data'],columns=cols) - - # add target to dataframe - data['attack_type'] = data_raw['target'] - - # specify and map attack types - attack_list = np.unique(data['attack_type']) - attack_category = ['dos','u2r','r2l','r2l','r2l','probe','dos','u2r','r2l','dos','probe','normal','u2r', - 'r2l','dos','probe','u2r','probe','dos','r2l','dos','r2l','r2l'] - - attack_types = {} - for i,j in zip(attack_list,attack_category): - attack_types[i] = j - - data['attack_category'] = 'normal' - for key,value in attack_types.items(): - data['attack_category'][data['attack_type'] == key] = value - - # define target - data['target'] = 0 - for t in target: - data['target'][data['attack_category'] == t] = 1 - - # define columns to be dropped - drop_cols = [] - for col in data.columns.values: - if col not in keep_cols: - drop_cols.append(col) - - if drop_cols!=[]: - data.drop(columns=drop_cols,inplace=True) - - return data - - -def sample_df(df,n): - """ Sample from df. """ - if n < df.shape[0]+1: - replace = False - else: - replace = True - return df.sample(n=n,replace=replace) - - -def generate_batch(data,n_samples,frac_outliers): - """ Generate random batch from data with fixed size and fraction of outliers. """ - - normal = data[data['target']==0] - outlier = data[data['target']==1] - - if n_samples==1: - n_outlier = np.random.binomial(1,frac_outliers) - n_normal = 1 - n_outlier - else: - n_normal = int((1-frac_outliers) * n_samples) - n_outlier = int(frac_outliers * n_samples) - - batch_normal = sample_df(normal,n_normal) - batch_outlier = sample_df(outlier,n_outlier) - - batch = pd.concat([batch_normal,batch_outlier]) - batch = batch.sample(frac=1).reset_index(drop=True) - - outlier_true = batch['target'].values - batch.drop(columns=['target'],inplace=True) - - return batch.values.astype('float'), outlier_true - -def flatten(x): - if isinstance(x, collections.Iterable): - return [a for i in x for a in flatten(i)] - else: - return [x] - -def performance(y_true,y_pred,roll_window=100): - """ Return a confusion matrix and calculate rolling accuracy, precision, recall, F1 and F2 scores. """ - - # confusion matrix - cm = confusion_matrix(y_true,y_pred,labels=[0,1]) - tn, fp, fn, tp = cm.ravel() - - # total scores - acc_tot = accuracy_score(y_true,y_pred) - prec_tot = precision_score(y_true,y_pred) - rec_tot = recall_score(y_true,y_pred) - f1_tot = f1_score(y_true,y_pred) - f2_tot = fbeta_score(y_true,y_pred,beta=2) - - # rolling scores - y_true_roll = y_true[-roll_window:] - y_pred_roll = y_pred[-roll_window:] - acc_roll = accuracy_score(y_true_roll,y_pred_roll) - prec_roll = precision_score(y_true_roll,y_pred_roll) - rec_roll = recall_score(y_true_roll,y_pred_roll) - f1_roll = f1_score(y_true_roll,y_pred_roll) - f2_roll = fbeta_score(y_true_roll,y_pred_roll,beta=2) - - scores = [tn, fp, fn, tp, acc_tot, prec_tot, rec_tot, f1_tot, f2_tot, - acc_roll, prec_roll, rec_roll, f1_roll, f2_roll] - - return scores - -def outlier_stats(y_true,y_pred,roll_window=100): - """ Calculate number and percentage of predicted and labeled outliers. """ - - y_pred_roll = np.sum(y_pred[-roll_window:]) - y_true_roll = np.sum(y_true[-roll_window:]) - y_pred_tot = np.sum(y_pred) - y_true_tot = np.sum(y_true) - - return y_pred_roll, y_true_roll, y_pred_tot, y_true_tot - -def get_payload(arr): - features = ["srv_count","serror_rate","srv_serror_rate","rerror_rate","srv_rerror_rate","same_srv_rate", - "diff_srv_rate","srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate", - "dst_host_diff_srv_rate","dst_host_same_src_port_rate","dst_host_srv_diff_host_rate", - "dst_host_serror_rate","dst_host_srv_serror_rate","dst_host_rerror_rate","dst_host_srv_rerror_rate"] - datadef = {"names":features,"ndarray":arr.tolist()} - payload = {"meta":{},"data":datadef} - return payload - -def rest_request_ambassador(deploymentName,namespace,request,endpoint="localhost:8003"): - response = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/predictions", - json=request) - print(response.status_code) - print(response.text) - return response.json() - -def send_feedback_rest(deploymentName,namespace,request,response,reward,truth,endpoint="localhost:8003"): - feedback = { - "request": request, - "response": response, - "reward": reward, - "truth": {"data":{"ndarray":truth.tolist()}} - } - ret = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/feedback", - json=feedback) - return diff --git a/components/outlier-detection/seq2seq-lstm/.s2i/environment b/components/outlier-detection/seq2seq-lstm/.s2i/environment deleted file mode 100644 index 867d00a693..0000000000 --- a/components/outlier-detection/seq2seq-lstm/.s2i/environment +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=OutlierSeq2SeqLSTM -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/components/outlier-detection/seq2seq-lstm/CoreSeq2SeqLSTM.py b/components/outlier-detection/seq2seq-lstm/CoreSeq2SeqLSTM.py deleted file mode 100644 index 32036198c5..0000000000 --- a/components/outlier-detection/seq2seq-lstm/CoreSeq2SeqLSTM.py +++ /dev/null @@ -1,215 +0,0 @@ -import logging -import numpy as np -import pickle -import random - -from model import model - -logger = logging.getLogger(__name__) - -class CoreSeq2SeqLSTM(object): - """ Outlier detection using a sequence-to-sequence (seq2seq) LSTM model. - - Parameters - ---------- - threshold (float): reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - reservoir_sampling : applies reservoir sampling to incoming data - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=0.003,reservoir_size=50000,model_name='seq2seq',load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.reservoir_size = reservoir_size - self.batch = [] - self.N = 0 # total sample count up until now for reservoir sampling - self.nb_outliers = 0 - - # load model architecture parameters - with open(load_path + model_name + '.pickle', 'rb') as f: - self.timesteps, self.n_features, encoder_dim, decoder_dim, output_activation = pickle.load(f) - - # instantiate model - self.s2s, self.enc, self.dec = model(self.n_features,encoder_dim=encoder_dim, - decoder_dim=decoder_dim,output_activation=output_activation) - self.s2s.load_weights(load_path + model_name + '_weights.h5') # load pretrained model weights - self.s2s._make_predict_function() - self.enc._make_predict_function() - self.dec._make_predict_function() - - # load data preprocessing info - with open(load_path + 'preprocess_' + model_name + '.pickle', 'rb') as f: - preprocess = pickle.load(f) - self.preprocess, self.clip, self.axis = preprocess[:3] - if self.preprocess=='minmax': - self.xmin, self.xmax = preprocess[3:5] - self.min, self.max = preprocess[5:] - elif self.preprocess=='standardized': - self.mu, self.sigma = preprocess[3:] - - - def reservoir_sampling(self,X,update_stand=False): - """ Keep batch of data in memory using reservoir sampling. """ - for item in X: - self.N+=1 - if len(self.batch) < self.reservoir_size: - self.batch.append(item) - else: - s = int(random.random() * self.N) - if s < self.reservoir_size: - self.batch[s] = item - - if update_stand: - if self.preprocess=='minmax': - self.xmin = np.array(self.batch).min(axis=self.axis) - self.xmax = np.array(self.batch).max(axis=self.axis) - elif self.preprocess=='standardized': - self.mu = np.array(self.batch).mean(axis=self.axis) - self.sigma = np.array(self.batch).std(axis=self.axis) - return - - - def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) - - - def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X - - - def decode_sequence(self,input_seq): - """ Feed output of encoder to decoder and make sequential predictions. """ - - # use encoder the get state vectors - states_value = self.enc.predict(input_seq) - - # generate initial target sequence - target_seq = input_seq[0,0,:].reshape((1,1,self.n_features)) - - # sequential prediction of time series - decoded_seq = np.zeros((1, self.timesteps, self.n_features)) - decoded_seq[0,0,:] = target_seq[0,0,:] - i = 1 - while i < self.timesteps: - - decoder_output = self.dec.predict([target_seq] + states_value) - - # update the target sequence - target_seq = np.zeros((1, 1, self.n_features)) - target_seq[0, 0, :] = decoder_output[0] - - # update output - decoded_seq[0, i, :] = decoder_output[0] - - # update states - states_value = decoder_output[1:] - - i+=1 - - return decoded_seq - - - def _get_preds(self,X): - """ Detect outliers if the reconstruction error is above the threshold. - - Parameters - ---------- - X : array-like - """ - - # clip data per feature - for col,clip in enumerate(self.clip): - X[:,:,col] = np.clip(X[:,:,col],-clip,clip) - - # update reservoir - if self.N < self.reservoir_size: - update_stand = False - else: - update_stand = True - - self.reservoir_sampling(X,update_stand=update_stand) - - # apply scaling - if self.preprocess=='minmax': - X = ((X - self.xmin) / (self.xmax - self.xmin)) * (self.max - self.min) + self.min - elif self.preprocess=='standardized': - X = (X - self.mu) / (self.sigma + 1e-10) - - # make predictions - n_obs = X.shape[0] - self.mse = np.zeros(n_obs) - for obs in range(n_obs): - input_seq = X[obs:obs+1,:,:] - decoded_seq = self.decode_sequence(input_seq) - self.mse[obs] = np.mean(np.power(input_seq[0,:,:] - decoded_seq[0,:,:], 2)) - self.prediction = np.array([1 if e > self.threshold else 0 for e in self.mse]).astype(int) - - return self.prediction - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return additional data as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - logger.info("Send feedback called") - return [] - - - def tags(self): - """ - Use predictions made within transform to add these as metadata - to the response. Tags will only be collected if the component is - used as an input-transformer. - """ - try: - return {"outlier-predictions": self.prediction_meta.tolist()} - except AttributeError: - logger.info("No metadata about outliers") - - - def metrics(self): - """ Return custom metrics averaged over the prediction batch. - """ - self.nb_outliers += np.sum(self.prediction) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":np.mean(self.prediction)} - mse = {"type":"GAUGE","key":"mse","value":np.mean(self.mse)} - nb_outliers = {"type":"GAUGE","key":"nb_outliers","value":int(self.nb_outliers)} - fraction_outliers = {"type":"GAUGE","key":"fraction_outliers","value":int(self.nb_outliers)/self.N} - obs = {"type":"GAUGE","key":"observation","value":self.N} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - return [is_outlier,mse,nb_outliers,fraction_outliers,obs,threshold] \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/OutlierSeq2SeqLSTM.py b/components/outlier-detection/seq2seq-lstm/OutlierSeq2SeqLSTM.py deleted file mode 100644 index 6dd72afe2d..0000000000 --- a/components/outlier-detection/seq2seq-lstm/OutlierSeq2SeqLSTM.py +++ /dev/null @@ -1,117 +0,0 @@ -import numpy as np - -from CoreSeq2SeqLSTM import CoreSeq2SeqLSTM -from utils import flatten, performance, outlier_stats - -class OutlierSeq2SeqLSTM(CoreSeq2SeqLSTM): - """ Outlier detection using a sequence-to-sequence (seq2seq) LSTM model. - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - def __init__(self,threshold=0.003,reservoir_size=50000,model_name='seq2seq',load_path='./models/'): - - super().__init__(threshold=threshold,reservoir_size=reservoir_size, - model_name=model_name,load_path=load_path) - - self._predictions = [] - self._labels = [] - self._mse = [] - self.roll_window = 100 - self.metric = [float('nan') for i in range(18)] - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return outlier labels as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - _ = super().send_feedback(X,feature_names,reward,truth) - - # historical reconstruction errors and predictions - self._mse.append(self.mse) - self._mse = flatten(self._mse) - self._predictions.append(self.prediction) - self._predictions = flatten(self._predictions) - - # target labels - self.label = truth - self._labels.append(self.label) - self._labels = flatten(self._labels) - - # performance metrics - scores = performance(self._labels,self._predictions,roll_window=self.roll_window) - stats = outlier_stats(self._labels,self._predictions,roll_window=self.roll_window) - - convert = flatten([scores,stats]) - metric = [] - for c in convert: # convert from np to native python type to jsonify - metric.append(np.asscalar(np.asarray(c))) - self.metric = metric - - return [] - - - def metrics(self): - """ Return custom metrics. - Printed with a delay of 1 prediction because the labels are returned in the feedback step. - """ - - if self.mse.shape[0]>1: - raise ValueError('Metrics can only handle single observations.') - - if self.N==1: - pred = float('nan') - err = float('nan') - y_true = float('nan') - else: - pred = int(self._predictions[-1]) - err = self._mse[-1] - y_true = int(self.label[0]) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":pred} - mse = {"type":"GAUGE","key":"mse","value":err} - obs = {"type":"GAUGE","key":"observation","value":self.N - 1} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - label = {"type":"GAUGE","key":"label","value":y_true} - - accuracy_tot = {"type":"GAUGE","key":"accuracy_tot","value":self.metric[4]} - precision_tot = {"type":"GAUGE","key":"precision_tot","value":self.metric[5]} - recall_tot = {"type":"GAUGE","key":"recall_tot","value":self.metric[6]} - f1_score_tot = {"type":"GAUGE","key":"f1_tot","value":self.metric[7]} - f2_score_tot = {"type":"GAUGE","key":"f2_tot","value":self.metric[8]} - - accuracy_roll = {"type":"GAUGE","key":"accuracy_roll","value":self.metric[9]} - precision_roll = {"type":"GAUGE","key":"precision_roll","value":self.metric[10]} - recall_roll = {"type":"GAUGE","key":"recall_roll","value":self.metric[11]} - f1_score_roll = {"type":"GAUGE","key":"f1_roll","value":self.metric[12]} - f2_score_roll = {"type":"GAUGE","key":"f2_roll","value":self.metric[13]} - - true_negative = {"type":"GAUGE","key":"true_negative","value":self.metric[0]} - false_positive = {"type":"GAUGE","key":"false_positive","value":self.metric[1]} - false_negative = {"type":"GAUGE","key":"false_negative","value":self.metric[2]} - true_positive = {"type":"GAUGE","key":"true_positive","value":self.metric[3]} - - nb_outliers_roll = {"type":"GAUGE","key":"nb_outliers_roll","value":self.metric[14]} - nb_labels_roll = {"type":"GAUGE","key":"nb_labels_roll","value":self.metric[15]} - nb_outliers_tot = {"type":"GAUGE","key":"nb_outliers_tot","value":self.metric[16]} - nb_labels_tot = {"type":"GAUGE","key":"nb_labels_tot","value":self.metric[17]} - - return [is_outlier,mse,obs,threshold,label, - accuracy_tot,precision_tot,recall_tot,f1_score_tot,f2_score_tot, - accuracy_roll,precision_roll,recall_roll,f1_score_roll,f2_score_roll, - true_negative,false_positive,false_negative,true_positive, - nb_outliers_roll,nb_labels_roll,nb_outliers_tot,nb_labels_tot] \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/README.md b/components/outlier-detection/seq2seq-lstm/README.md deleted file mode 100644 index fa18b1d4d1..0000000000 --- a/components/outlier-detection/seq2seq-lstm/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Sequence-to-Sequence LSTM (seq2seq-LSTM) Outlier Detector - -## Description - -[Anomaly or outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. - -The implemented seq2seq outlier detector aims to predict anomalies in a sequence of input features. The model can be trained in an unsupervised or semi-supervised way, which is helpful since labeled training data is often scarce. The outlier detector predicts whether the input features represent normal behaviour or not, dependent on a threshold level set by the user. - -## Implementation - -The architecture of the seq2seq model is defined in ```model.py``` and it is trained by running the ```train.py``` script. The ```OutlierSeq2SeqLSTM``` class loads a pre-trained model and makes predictions on new data. - -A detailed explanation of the implementation and usage of the seq2seq model as an outlier detector can be found in the [seq2seq documentation](./doc.md). - -## Running on Seldon - -An end-to-end example running a seq2seq outlier detector on GCP or Minikube using Seldon to identify anomalies in ECGs is available [here](./seq2seq_lstm.ipynb). - -Docker images to use the generic Mahalanobis outlier detector as a model or transformer can be found on Docker Hub: -* [seldonio/outlier-s2s-lstm-model](https://hub.docker.com/r/seldonio/outlier-s2s-lstm-model) -* [seldonio/outlier-s2s-lstm-transformer](https://hub.docker.com/r/seldonio/outlier-s2s-lstm-transformer) - -A model docker image specific for the demo is also available: -* [seldonio/outlier-s2s-lstm-model-demo](https://hub.docker.com/r/seldonio/outlier-s2s-lstm-model-demo) \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/__init__.py b/components/outlier-detection/seq2seq-lstm/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/seq2seq-lstm/data/.keep b/components/outlier-detection/seq2seq-lstm/data/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/seq2seq-lstm/doc.md b/components/outlier-detection/seq2seq-lstm/doc.md deleted file mode 100644 index d1911d311f..0000000000 --- a/components/outlier-detection/seq2seq-lstm/doc.md +++ /dev/null @@ -1,336 +0,0 @@ -# Sequence-to-Sequence LSTM (seq2seq-LSTM) Outlier Algorithm Documentation - -The aim of this document is to explain the seq2seq-LSTM algorithm in Seldon's outlier detection framework. - -First, we provide a high level overview of the algorithm and the use case, then we will give a detailed explanation of the implementation. - -## Overview - -Outlier detection has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The available data is typically unlabeled and detection needs to be done in real-time. The outlier detector can be used as a standalone algorithm, or to detect anomalies in the input data of another predictive model. - -The seq2seq-LSTM outlier detection algorithm is suitable for time series data and predicts whether a sequence of input features is an outlier or not, dependent on a threshold level set by the user. The algorithm needs to be pretrained first on a batch of -preferably- inliers. - -As observations arrive, the algorithm will: -- clip and scale the input features -- first encode, and then sequentially decode the input time series data in an attempt to reconstruct the initial observations -- compute a reconstruction error between the output of the decoder and the input data -- predict that the observation is an outlier if the error is larger than the threshold level - -## Why Sequence-to-Sequence Models? - -Seq2seq models convert sequences from one domain into sequences in another domain. A typical example would be sentence translation between different languages. A seq2seq model consists of 2 main building blocks: an encoder and a decoder. The encoder processes the input sequence and initializes the decoder. The decoder then makes sequential predictions for the output sequence. In our case, the decoder aims to reconstruct the input sequence. Both the encoder and decoder are typically implemented with recurrent or 1D convolutional neural networks. Our implementation uses a type of recurrent neural network called LSTM networks. An excellent explanation of how LSTM units work is available [here](http://colah.github.io/posts/2015-08-Understanding-LSTMs/). The loss function to be minimized with stochastic gradient descent is the mean squared error between the input and output sequence, and is called the reconstruction error. - -If we train the seq2seq model with inliers, it will be able to replicate new inlier data well with a low reconstruction error. However, if outliers are fed to the seq2seq model, the reconstruction error becomes large and we can classify the sequence as an anomaly. - -## Implementation - -The implementation is inspired by [this blog post](https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html). - -### 1. Building the seq2seq-LSTM Model - -The seq2seq model definition in ```model.py``` takes 4 arguments that define the architecture: -- the number of features in the input -- a list with the number of units per [bidirectional](https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks) LSTM layer in the encoder -- a list with the number of units per LSTM layer in the decoder -- the output activation type for the dense output layer on top of the last LSTM unit in the decoder - -``` python -def model(n_features, encoder_dim = [20], decoder_dim = [20], dropout=0., learning_rate=.001, - loss='mean_squared_error', output_activation='sigmoid'): - """ Build seq2seq model. - - Arguments: - - n_features (int): number of features in the data - - encoder_dim (list): list with number of units per encoder layer - - decoder_dim (list): list with number of units per decoder layer - - dropout (float): dropout for LSTM units - - learning_rate (float): learning rate used during training - - loss (str): loss function used - - output_activation (str): activation type for the dense output layer in the decoder - """ -``` - -First, we define the bidirectional LSTM layers in the encoder and keep the state of the last LSTM unit to initialise the decoder: - -```python -# add encoder hidden layers -encoder_lstm = [] -for i in range(enc_dim-1): - encoder_lstm.append(Bidirectional(LSTM(encoder_dim[i], dropout=dropout, - return_sequences=True,name='encoder_lstm_' + str(i)))) - encoder_hidden = encoder_lstm[i](encoder_hidden) - -encoder_lstm.append(Bidirectional(LSTM(encoder_dim[-1], dropout=dropout, return_state=True, - name='encoder_lstm_' + str(enc_dim-1)))) -encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm[-1](encoder_hidden) - -# only need to keep encoder states -state_h = Concatenate()([forward_h, backward_h]) -state_c = Concatenate()([forward_c, backward_c]) -encoder_states = [state_h, state_c] -``` - -We can then define the LSTM units in the decoder, with the states initialised by the encoder: - -```python -# initialise decoder states with encoder states -decoder_lstm = [] -for i in range(dec_dim): - decoder_lstm.append(LSTM(decoder_dim[i], dropout=dropout, return_sequences=True, - return_state=True, name='decoder_lstm_' + str(i))) - decoder_hidden, _, _ = decoder_lstm[i](decoder_hidden, initial_state=encoder_states) -``` - -We add a dense layer with output activation of choice on top of the last LSTM layer in the decoder and compile the model: - -```python -# add linear layer on top of LSTM -decoder_dense = Dense(n_features, activation=output_activation, name='dense_output') -decoder_outputs = decoder_dense(decoder_hidden) - -# define seq2seq model -model = Model([encoder_inputs, decoder_inputs], decoder_outputs) -optimizer = Adam(lr=learning_rate) -model.compile(optimizer=optimizer, loss=loss) -``` - -The decoder predictions are sequential and we only need the encoder states to initialise the decoder for the first item in the sequence. From then on, the output and state of the decoder at each step in the sequence is used to predict the next item. As a result, we define separate encoder and decoder models for the prediction stage: - -```python -# define encoder model returning encoder states -encoder_model = Model(encoder_inputs, encoder_states * dec_dim) - -# define decoder model -# need state inputs for each LSTM layer -decoder_states_inputs = [] -for i in range(dec_dim): - decoder_state_input_h = Input(shape=(decoder_dim[i],), name='decoder_state_input_h_' + str(i)) - decoder_state_input_c = Input(shape=(decoder_dim[i],), name='decoder_state_input_c_' + str(i)) - decoder_states_inputs.append([decoder_state_input_h, decoder_state_input_c]) -decoder_states_inputs = [state for states in decoder_states_inputs for state in states] - -decoder_inference = decoder_inputs -decoder_states = [] -for i in range(dec_dim): - decoder_inference, state_h, state_c = decoder_lstm[i](decoder_inference, - initial_state=decoder_states_inputs[2*i:2*i+2]) - decoder_states.append([state_h,state_c]) -decoder_states = [state for states in decoder_states for state in states] - -decoder_outputs = decoder_dense(decoder_inference) -decoder_model = Model([decoder_inputs] + decoder_states_inputs, - [decoder_outputs] + decoder_states) -``` - -### 2. Training the model - -The seq2seq-LSTM model can be trained on a batch of -ideally- inliers by running the ```train.py``` script with the desired hyperparameters. The example below trains the model on the first 2628 ECG's of the ECG5000 dataset. The input/output sequence has a length of 140, the encoder has 1 bidirectional LSTM layer with 20 units, and the decoder consists of 1 LSTM layer with 40 units. This has to be 2x the number of units of the bidirectional encoder because both the forward and backward encoder states are used to initialise the decoder. Feature-wise minmax scaling between 0 and 1 is applied to the input sequence so we can use a sigmoid activation in the decoder's output layer. - -```python -!python train.py \ ---dataset './data/ECG5000_TEST.arff' \ ---data_range 0 2627 \ ---minmax \ ---timesteps 140 \ ---encoder_dim 20 \ ---decoder_dim 40 \ ---output_activation 'sigmoid' \ ---dropout 0 \ ---learning_rate 0.005 \ ---loss 'mean_squared_error' \ ---epochs 100 \ ---batch_size 32 \ ---validation_split 0.2 \ ---model_name 'seq2seq' \ ---print_progress \ ---save \ ---save_path './models/' -``` - -The model weights and hyperparameters are saved in the folder specified by "save_path". - -### 3. Making predictions - -In order to make predictions, which can then be served by Seldon Core, the pre-trained model weights and hyperparameters are loaded when defining an OutlierSeq2SeqLSTM object. The "threshold" argument defines above which reconstruction error a sample is classified as an outlier. The threshold is a key hyperparameter and needs to be picked carefully for each application. The OutlierSeq2SeqLSTM class inherits from the CoreSeq2SeqLSTM class in ```CoreSeq2SeqLSTM.py```. - -```python -class CoreSeq2SeqLSTM(object): - """ Outlier detection using a sequence-to-sequence (seq2seq) LSTM model. - - Parameters - ---------- - threshold (float): reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - reservoir_sampling : applies reservoir sampling to incoming data - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=0.003,reservoir_size=50000,model_name='seq2seq',load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.reservoir_size = reservoir_size - self.batch = [] - self.N = 0 # total sample count up until now for reservoir sampling - self.nb_outliers = 0 - - # load model architecture parameters - with open(load_path + model_name + '.pickle', 'rb') as f: - self.timesteps, self.n_features, encoder_dim, decoder_dim, output_activation = pickle.load(f) - - # instantiate model - self.s2s, self.enc, self.dec = model(self.n_features,encoder_dim=encoder_dim, - decoder_dim=decoder_dim,output_activation=output_activation) - self.s2s.load_weights(load_path + model_name + '_weights.h5') # load pretrained model weights - self.s2s._make_predict_function() - self.enc._make_predict_function() - self.dec._make_predict_function() - - # load data preprocessing info - with open(load_path + 'preprocess_' + model_name + '.pickle', 'rb') as f: - preprocess = pickle.load(f) - self.preprocess, self.clip, self.axis = preprocess[:3] - if self.preprocess=='minmax': - self.xmin, self.xmax = preprocess[3:5] - self.min, self.max = preprocess[5:] - elif self.preprocess=='standardized': - self.mu, self.sigma = preprocess[3:] -``` - -```python -class OutlierSeq2SeqLSTM(CoreSeq2SeqLSTM): - """ Outlier detection using a sequence-to-sequence (seq2seq) LSTM model. - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - def __init__(self,threshold=0.003,reservoir_size=50000,model_name='seq2seq',load_path='./models/'): - - super().__init__(threshold=threshold,reservoir_size=reservoir_size, - model_name=model_name,load_path=load_path) -``` - -The actual outlier detection is done by the ```_get_preds``` method which is invoked by ```predict``` or ```transform_input``` dependent on whether the detector is defined as respectively a model or a transformer. - -```python -def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) -``` - -```python -def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X -``` - -First the data is (optionally) clipped. If the number of observations fed to the outlier detector up until now is at least equal to the defined reservoir size, the feature-wise scaling parameters are updated using the observations in the reservoir. The reservoir is updated each observation using reservoir sampling. We can then scale the input data. - -```python -# clip data per feature -for col,clip in enumerate(self.clip): - X[:,:,col] = np.clip(X[:,:,col],-clip,clip) - -# update reservoir -if self.N < self.reservoir_size: - update_stand = False -else: - update_stand = True - -self.reservoir_sampling(X,update_stand=update_stand) - -# apply scaling -if self.preprocess=='minmax': - X = ((X - self.xmin) / (self.xmax - self.xmin)) * (self.max - self.min) + self.min -elif self.preprocess=='standardized': - X = (X - self.mu) / (self.sigma + 1e-10) -``` - -We then make predictions using the ```decode_sequence``` function and calculate the mean squared error between the input and output sequences. If this value is above the threshold, an outlier is predicted. - -```python -# make predictions -n_obs = X.shape[0] -self.mse = np.zeros(n_obs) -for obs in range(n_obs): - input_seq = X[obs:obs+1,:,:] - decoded_seq = self.decode_sequence(input_seq) - self.mse[obs] = np.mean(np.power(input_seq[0,:,:] - decoded_seq[0,:,:], 2)) -self.prediction = np.array([1 if e > self.threshold else 0 for e in self.mse]).astype(int) -``` - -The ```decode_sequence``` function takes an input sequence and uses the encoder model to retrieve the state vectors of the last LSTM layer in the encoder so they can be used to initialise the LSTM layers in the decoder. The feature values of the first step in the input sequence are used to initialise the output sequence. We can then use the decoder model to make sequential predictions for the output sequence. At each step, we use the previous step's output value and state as decoder inputs. - -```python -def decode_sequence(self,input_seq): - """ Feed output of encoder to decoder and make sequential predictions. """ - - # use encoder the get state vectors - states_value = self.enc.predict(input_seq) - - # generate initial target sequence - target_seq = input_seq[0,0,:].reshape((1,1,self.n_features)) - - # sequential prediction of time series - decoded_seq = np.zeros((1, self.timesteps, self.n_features)) - decoded_seq[0,0,:] = target_seq[0,0,:] - i = 1 - while i < self.timesteps: - - decoder_output = self.dec.predict([target_seq] + states_value) - - # update the target sequence - target_seq = np.zeros((1, 1, self.n_features)) - target_seq[0, 0, :] = decoder_output[0] - - # update output - decoded_seq[0, i, :] = decoder_output[0] - - # update states - states_value = decoder_output[1:] - - i+=1 - - return decoded_seq -``` - -## References - -Francois Chollet. A ten-minute introduction to sequence-to-sequence learning in Keras -- https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html - -Christopher Olah. Understanding LSTM Networks -- http://colah.github.io/posts/2015-08-Understanding-LSTMs/ - -Ilya Sutskever, Oriol Vinyals and Quoc V. Le. Sequence to Sequence Learning with Neural Networks. 2014 -- https://arxiv.org/abs/1409.3215 \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/images/ecg.png b/components/outlier-detection/seq2seq-lstm/images/ecg.png deleted file mode 100644 index 5947a605a2f5ff76663f0afa4dd0722d384130a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35448 zcmafaWl$Z#7AAz??k*RCyE`Nh+}%C6yIXJ%aIxU-?ry=|-GjT!MfT>sdbRs+yJl)? zq`S|Y?mqf;grd9zG6Det1Ox=Kl;n412na|w@CgS81AZdowlxpFe0CO*QiTH-AGjak z;CpxnNiAmx2tpL_A1Q~?X_iw5m8E2X99@uIxL&Aqmcbig@ z#V{h5YQLd1zbYx6L5tN^v{|pbwVsxhUsQE6<>j?RBBOmSpb_&5!y*I3-09no+8#a* zt$(GlL=*XW>zM`mYBrhSX@0SO$$Oji*dZZFtMsq5I!)`^L8<;Lt2k=QYT#=Xi>i^v zzgK;VA$nqBV*3c9*lyU^*h%zA|IY`|U;n~4x ziG<0C%7c=$+-o01$g(gZ1whes{-MGm$~>FytvCe0ZU~w<;CnLKEsY0 z=rZis4N(5U0D&gr0mTnVZ5RxkMHYh-45R_oUP8n{DSQ(8%5tT#{hOGM-R}e^)wnIVC5>wa&_2Fh&Jco11mPIlJPs;BZMaaRpRCV@x;(3_$T3h+*i9rpt8LRLI1)*YZi{n2oas6x&^WW*Ggtk1fO}c$&}B z=O3t+7v7M0fs9{y6+ipTZKoDp47q;I>+G`GE**RGDbKzYph#JRxSvBYaGLAuLiEUu zFd_YZe9fiSji?BDSMx#q`45Nk1L=eF3zT0E%C;D&>lVR1$b=B{K@#+!Z*EYnx7@_3 z9=7oyyKVOs7O4SIFWUFRxcw zYr%Cuto!YxK+_2ND+V_;%8FXH2qh|O7XV=cNCQ@oM_*|Q*O>|%AYFUjdPoa_l**&h z+u2|HuVL6nIe!-dtrfiv1+^^{oVSMXC26Ayfx?os<>yTuNYU1c1+|w0(Yr+_yT1f`8E?v?Xjr}O5W%Xi zOWgT)T+ShAJ74mcvO8gTJ|;1Yxw(MJD*^rQDwG!wX zCR>~9Z!M7fj9*0IYQ^AK^9q~F8c54(%56TI!2<}qar~e~g3*G~%f3su%1iU&iveRH zNx79dV8sMZ;P+kPKq6@&yHi4bkV9slbSACtbPIvEgh0#GPmORT-yG%UQyzbf8rsQN zFlC-FH*EWHZ%YB21C5lW-3=>z+XDu>acNDnfH!`$rnuXX+fab-924uPwEOzB`$$+C zs|ed`IIE?zu_b*~H!#~MKKhsH=pvsH0h?$df23*Spgy3J5dc((fQ++v0L3Xin#gB3 zz`G%8T!pBkpe7;j)>4l_2%4V=oWZu|_821Ty5A;r&sKo>c~!#V4d%2SoMSthi2Vl= zy2u@$ff0yT2xb#Yy*Ods17@70x=&1^Z+TLnCFPH+}s$-1_++%POnBK1TW_2{#vu`3$%AlTnL zt7{kLnsp+5p{5Z!VFUb$6dx5p=z#4&1wzu&*i)8Isyr|P6&J{X-;T2h&#X_p@qga< zYUCn)7N#ncweEduA(m`5cns>XW$ds^a{De>%G8jXmzU>6nfvhn(-3+(YCli|*I<9h z&=zy5EiNr-gocGRFO>dMEPl9WTixA~(#K1>tz4LHKpPhcCF8vGF9HG?zMzN3o&S7P zc3*^c-w=EVcbn;br~6lMC3WtWB_RFLL<*o}0^vo%5l7$t z{a2pTwQfU=3~hFja;PM2ztWA&D_x=Lt;D}pb*big{;T|=n~>bU)6w_a+l}%4`x{kl ziQ<#m4=0XnRIY4`uAIoo$kEGGdCD`L1O%-X&U>fye{Ij^9;^0UtW4QuPEA{kD7uS(G{(%f2J2O)>Xi!(31K?*VZNxIdKpzAEv-f5dV~Q)9Xg9gQJYTs+a#v3WCB!`1ajMCFi>Gf27D4gPT()`#hhfM*5#_O zsn1|C#gW}rl`JJzP3wzq;oJ4kYs*U-aKjK# z&`$}Y7NgTHn-FliF-0inNokCB#?mNL>7E4C4V1%aeA!!!5?LNWc9!^+ula9>p@ ze_2TbtVCq2VItSd^CQ1WyL|rM0&8{NxBl*j_fs~lrP>l1UdRB^AD{W`?OncoE30XS zp`;VP2NSDng@Q{YPQmf#WiYG0I?5XN;4Y(ov7+*?>b@?_DIZP|ChbvB&2J zIQGM$;k4yEr-GX5YG4pF?D4@0vPRIU=i}D$n>a+5%uAKv#L2Ju!UDtaSibva6WMW7 z;;-i}eN}D7s;W$a>gh=oViHG}Tir8k->SYvBjOL${WVRkC~|A4G668+$XZ(Rd>6}J zbsL?^^0=#8o__|fmPr9MrPlcj8vei;8yn?i$b-97Q7Kz^LN>03L{nl^XLX#}AANw6 zaT0#rF39;w>a&w^73?IL$=si!5OGzL_D5x_b@*;9x4!lF;x(>>VE zt+0yxxi!|MD!2bQp^Sn{GPpF7Vz{(THD(zc>J8eI@o^Neo?C26bR~|O%Z*hsw{ed? zmb3MF-DfFfM`&mBc=0|bE&($iegOXN`aB3NZ>5es&un_E9oPQMKZrm^j$d=BmZdP` zD=uF6;{EtHvqMNIR?{Dblgt+G>3rEWrwguNUNTXN*T*;9aCm|4?36q!#mDAEv#GlI zvInlCm@6HLf3;!Knm;X_I^weu1)OZOudWry)0{o*U;ne@TRDgrg(AOW>|xIBn!rFK z8A~r?3B`N>1xz+6|8dop>qJ>)>Z*EaZ@QjZ8x=LW-L1J@50~*wB)m!_TuvlBUnKl9 z;E#!|k){dW32uWrg$a15;Nb^G0}N!m&+Y|SpWweFa6fi?(vF|}N@Zt{=j6mApTc&_ zo!J(!!5h@$U)yk9#}RZxyea_J;j$2UXu{33QRiXv%p%PzMYpCD0w!nLb6dOOpBmAdtnX9 z98+2DWJ}s>zoi*rEdx}Z|LJ=5O|_v~ET}X(t}TSXWjKRa64#At&gULWNNG%pAP&7* zC?9$2KCA^MD>vET_pRVRmKSFz9c%<$-bk%eL%;zX@P(pJ^xET_9_ASIbX3WMr}M*K zJ_>*-cWbJ8tQsgo?uWJK?LVx_$KhhWTJ)GD87|%C(4bPS&mc}g9FAEZVNYJHZ}3YN6%`v6?mFDb8c*{^L+qr8*{q^c#}|_QW(mw zk6$o^eLZ$4aee20unG%eyR57o_rIh&=;qGi{on?&a=RCQSsI>AdHB_$Cx_@wyOPu% z6$lCCX)|rITo7ZpWGFx^$XqsgnqNIcSNZC(JH%CYa|4(#yU~BAhW>cxwp-E{eVVPF z;o&1MFTd{Ck>!>(Go`{VLWnk_NU!ID+7>cpy}TFAp9zp@&InKW*q!q1YkrKS{@N4-|FXq>j;h3h>csLx=zcs3LQzw6}%zn5Z?>LkR zWYYS%MBI$|n*Fgo-z|(j=0|qsZ?G$1F^#{!7r6`JFwIxPXUYlDXV`Pvw8nC={O$aD z@7&lO%uSm_WdvH&3SeFLTYV8@G(Tw>eUY;X=bGk+7F3 zO0xA#bmh(1dIls(%Nf*GfYwHPYz@sJp=fj_$(KjqJ!t|WJq3p-5{LDQ|*J` zaOTM3W(9k*?-M=bj1@8lumOwxP*bPX#Xb|i-A0J#{;NICEAYlnmz$|F!~MEk&i6NV z_cGRzWc?e2y@HZSxA?w0x-C#P+&l&W6E-~C@dDz^Q&;c za-g@Ww*Xk&5Yl=c>Ub#6cG*qpn?lyt{r=M7X*e0SVs>CLqGar2a&)nfX)$HtC;7fJ zPyS`oeKQbDQ=+f$yo}T#rkm@5WwGhf4OWFZVZNCJx4{#Gz-vb8^%De#c5fs%Dn>Ie zV)+Y_F4i&j@n8H~JRF{ig`uvF0~+zfB+)ce8!0Ovedz*;FQulXxF%-p%9ywki~Wc! z!6E1`1Gq;}pR|3>B4#n?wN8&t*Kh%!AMZQkOelvutj1JQZOxGM+GlNVsdU9-JAyTG zrr|^Mj^*5(3P^wGr_(BB5;br9`BxhL(nPE| zmsGMf*A8e1>C`uUV&N-r1$e9}VAY;DprFRYkuT3hG;>jLuUg05M3YwvAeUF}bsMGE z*N=4glMs-S(vf<^+*tXR#|i@z!Uu~A;}l3Cip1sj({iV3Xq_>q-X4gQrOV1!4}}_} zh`hax6}`_gQ7jW}KwS9d?odHh$n7O<0#F%g_kX|E#ils@O_5bb!AWQlOxfy^E1Ft1 zgpoxVSlO5sPc~SwnOv7`Fn6hsvgG(CGB9OZ|DZp1t2+*GtXcmaA91)8?^}G1f zFBvB(%Z0h3wi4Wy?Z4_Pk?V4-3>IfRT@SQtU6W!JBNT2vaH1b*Gk?kaPjW?2Yz3r7 z;F*g&g(Y!Q<{q8xs#~!Zuxe@S(3?RMCLnJuUBPP9_%~*s1T&c<7eTj5@th@ug`v#| z$<;&&umqtn6XSe_vZNJ(YkRmeJL!LGc@#EjIc^1f|MTMHu_x8_cBmg&r%LMM3|vZb zuGuErYmc>GaCBVRFTf)3v@zx;y4N;CMF4;~&AmuEWX9jTL~@J||JBNK$ZbfnJn4p1 zW0%j4=veDhW>B(z{!g9`+C)YJPLjM-zk={a(g7oeatiJzJU!{Q`p4o1_R%K`<8OWM z*jhX-M{vi-uBFUi3zmO5Wk@C8m=D@@!OV?8p;vf3hNS<~T8CdQ`cp_7>;5L5osS!w zN5E9Y0Zra{&3wZ!*C5vk2|1<##UdeIe|1mZE3t1EYM_ZK(6 zvlqbXX408Ky5nF+)Dsf%PN8p!5G_7mF)#a<>uB#024HCR^<2}D< zGcttp{kh@kdd_MkYyQr(fn@i@4|0DZ@m{&vw-6MhMU5_8w4n0(HS2Jny^NiUO|4_z1Oxj9cNIe@%@tmYYxmN`QkA7Tc>1nnr zYj}igO=?E5`Lw1bJ#a4UFEehF_ z5&e{SM2*3NG&?O!StxgSIKwlBpsdq_ zq1`hh!+VefPZNKfT|}eD!b{@jErhl$NYd^Q8|-$@<T)Ak2|}5a zHP#&T^MK`QLA!T;>mR1n+ntmBVN#NF)Sl61R3$a-r3)?>Rk!^M5&8zCOQ&}=e*zP0 zCx9Ia-;2AT|IZqJ@n98`rW>7`G{^IJlK6Q|E50-B61DW^98}~&O+rb%#(f}R5lQt| zTrVWVzuTdSMDLV|U~7)&#u^7ek@+lFWgG^e4H9)n^ovn$VDTO+gN@M zSMdfx^Caz=SIqCY&LDEzgqUnfx%_E&+mvx405K}*yhOfjvsne`bLrAsARr7DkT>EnTc&PeYmqg6&IA7&~yI`cGMIGwTP)LrP0Gl<(au|<2q+s}pb zmLU3C$?JosGWtXfPYM5RX_F6ABmE-@HW2cvnb4exUuWAzGlv^?~7 z0sj(c-A@8e%D`roT~iwC8qhkHIdbR+d!zR966Ieg2Q}0Wsxi3>VoA%!v^V@>iUhD- zdF^<&qd8KmuI+bRy4pqDpsgE))S+JXgMT<;5_%;%~aYd3sn ztf2|x4zg=~4ST*m>|dTbw64VPyKT=mGxsgT%&uE?q@z7}1*82?H_Jo+tzu7cNYPjm zNl>5EYosq(538N4;(aD(q}Wf)k5oi}SS}n+;b+YSMHS{IsZ>oZu$U6z{y6`64Zl6N_w7Snx1sP%)=Z@~fVRCNltOD?cdhst-pnr6D^%j82EYmq zxRQd+T>%pktt2g67vhvp)}FkLj@K>G1Y%-F$m&tZmrOD3Y6J-?NkQH9!d|v}&p%>+ z&^n$$^>;upX1zejtCtBrh%5P?5kp0-jKJ#ZlcZQ?BR3?F&h~TR-y{F zgkbwFK+w}$+x0>w!g0y2X2k)&@M#Is=cf)=PKI96hD~f{%b8OKvbGmAoik4d(l@KP zPTF~gq1fK0OL@g#N+X=NFKFyV`w<09+;7w&v+lYqVO(YjPFJIPPSumzL!-bXS)|Ni zLEj4_gLk25Gn3N|mTS*DBo~UlX45>4%djn<*SpS`tMHzsk!BNkP}A0QOWQ7ZpPXMIoa{AI!AJKV4VFU0cTYZnq5<{fA_{&gudmM#V*n#=(?@BX-GQ{6l z4>wlq61Q}CI6KaC6qV^*5cw;TQs;$JCiA=@oiTc>VD*GRXSG(*MW1YoQNsxLa_%gl9nCt$ zzh5R}6%?rTp|Di73%AcnPYx5af3biI;S4FzC}=LLlKHkWTR8lipyRTYFkqL+B=IUePfqTXxZnd*xGC=KLao;uyX94y(^5ADk?^=bfCCt0kraX(6}}Ka%)sx zS@?JCV~5QRDR!eB?3mF^Rok)6*8;#sCxqO?vU?%UUe4Tfb@^c_Nqcz_Vt!k+2)%YQ z(oV?qG0rMP;gDUU6oWcp8y8w~f12r2E8kWByG_@oa~mfy`f-0pZG80Q&e9Gn^jcKp zF2SzcTG}K7z5nNF3#i6;-1sqs-PcR?ekx7sLA8+Zcp964f!`S zC;GqnWTd^dU+qQ7Ua zn=`%4*sm-gS{Z54lbl2;7ga%={jdf)y zR+6u7aqTYj@uTvFfIWg2TYxQ>Fh{rfTe;=Zz5xHecf&Vt?Pca+Rl%MlG-h)< zqV*&~n2@V0wHceYTeS|3dem#)ecjs!`wox{8+aN7XE>R4G{>|?xz#h}nox1IVPTo6 zJNcM1`3}PFoh-D5R0ESUoBoFd01(7%pMZ=V+_&xokT;RVK^ZIvt7R!S|CZ6wk99t#WVY zt01%o2^N~!5{{F|j%wVvx+6O9n`i5n?=&WH>n*stZHJk(dUt|fsAB#$kA~gn3x{e<(yYi-pDd` z)Pw51M!kubj;zcrdJ*sZC zp4^+-pmzS7_e!W%BSWE&W^zbjZrJ2?ek4d)e|JK6zqT+xJwbz(bFM50v(44EMO&Ro__q&6xfRDW|n5LHCvuC|#FD}=<-V&&%MCYJi5lD-M~>c{$E z8>_T6orl+{c!t1RnsV9x_N31^yYb{vt)1jJb^h}3{4&{30VX!qIX3(6f1fW;f9Pw#id?1SUdx8*vV`aqSW z{-f=g;&=J+QnJ-8>>znjulRUu*!pAp$Lb}M=_dQQP89_2H)QK592VN{1WDl(oaYnj zJ0!#L?AKojDij-}8$#i@{?cD)yCX63BlcnI)8^>|meEk9qp1en9!*820-~DgD@&_b z9Bnk1pe?b%PN0%C$DD1AuwW?c4)&2O*r$J}k~e=og;gMC-+iHJ``+wB_p-lxOd4Cm zUYMGMD4uyurLJXtuv3jxs8=&K58dkr~sERt@}a&*m8Ezx)N1u(}?cf5pCdR z?mRJE6+nK1%CHh9*EePzi$S<6RsEG4Cd}Kpzh`?9xoSzh=LUBBi2x2C0rR~K?sU=} zV0nJ*)U{!>&W`rT=;Ka(mxf&bj<{Oy*HoImGX$+z)nWSO^Y=jZn8)Qx92z0Wu>M@6 zA|(|X9H+|sS^4-<1*=KO^78O%Ein=4D|*JhSnK!Rmz7_oa$!5$ku0>kt2CM-T5tfF z4dgc>FrOx}74)p=8sQ`6slFI8G7o{O+QAX01YUcRgA82;Pfu05<#5qzeUHiCQUj~m zrMym`Qyfib-#yS5K&WKa)}?Z?e@q6Eh`b+#Oq(aH@+)$F*9Zw6>dtv~^T{xR)q1fA z@zUTv+lN0a+@s^cdu@D2Co6uG+)eXw=M>LmjCD7^)^JY~xxfl9w$=KV23C z^Prp5?uT2}CQ4k4^UKkLJ!$+-b;AXmJ&aaDfX(0fH1CVwQ+bC?n2#m|8gXz&{^GQV zU3@)6ABh42J@GMWMipaqjyyuK9dE}-^258q$8h;=6FruHKCxshfa-r^111y{y|yFh zb2*5s-f)pJ-P@=IUcU`11h2A96Ob8Kkmv2aBXxH|rt5#QCSJdyJwl31H=7%Cc(T5V z{mE+}ss-a1Cj6a(vQrkhPE-AFPg^%lOvC`mD1F1UEgMn?W=CHsZ(G(3{8;Ljaffw@ z<3fcedLFO7O|hEYZ_AIgmyYYBk zC(`7z=h>KYIwx=;7zZ2cmP9VLb0*bg^}qDg`x!;sn?kgBP8tP#)HohvBEFKb=-94z zI8dX0TtaM(6MozJ<&621P0mch;EINYDE9@YO_R&lGaBs2!(E>C4-@94XNa61`2LRG*r0PxJN!G8+!HPw(-iJ_#K0Pkud_g7?(4NieTEAF&oD-D@><>~QjBkf&IaI z@Q!~p?G|$Sjyw<*0!B0Ca8d;_Yn=B6(tia6kJJV0B^NeIw0z52@QR=gA}1vfe0u!J zuTImqfdeNVtz4=0+gPtGBSgW$%_QxpxlF<#XpXm+>}G1vv?7`W&Fy>IT(FxOD*2ykRl$F^qUQXyi+U8taF& z6-JuJ#fmPfiN|+27Zmf2xik1S01}*n-PP=ZANsw4*Y;#m&Z$InDQRhit*7q^ow74Z zr>O6Iu+3?X(m0g(3x{2vmDSwFKDti8bkWgT89spg%)UY)FQ+ax&W`Z=mLclmpF`H1 za%#P)y+t+eGsd24)t0@1`{l%RJf2Rx6G9TOR48^wHd+Mnx7^kXGPJtWuJg)a& zosRwLkbl;>M9QVe8EQnECq=;M)r1JTggz9 z+1tc37Wd_QPRF~K?o}m0H@X9xpV9YNm|ifovd=|NA>NnVgR)c{30NV0o9zxO-EaQD zBdyu)Sv*od>GW`s&-wjmX6S!8Ua%=GGGtQ*%(tP9F(^v@?LYKqkrR@X-9g9T-ATbigX=aM5>c*0!;E=7R@o1(>); zMVLzt0?OUIw$DzeB9nl^nn`rGF4%>E(HuQR;c5PF2P8CnC~tNH>$VteK*sPS)P8Ce z!-rxtE`+9c#Dqn9+bG(v%Lr(erv0`Jw(&&scM7mbW$&-PK3 z6Fk|V{oT!p1YZgLgKAfRkp01?Xm~Q;?qbel84u#&rBAkpsN77x7oc9+t&IcLx~Hu^ zCsA?huYm~igCzD!od0%*3H4D`uAgIxr@U>U**4v%?P6?>F#l+|Wk6A%C@j0g^pI_f z^5yi@EQ~io4j%4);`#l`A#hUlZLcd^SX#o4pf2(%@7#iTW9B|h{^JUwLzhW$e5wev zFnTCZkW_qq(d$lIO<(LktcmwXYC7cUvum!Rs5G{Dhe49=;pbRY{IV>?Cd?lVVEA1X zg_DbFu{m(`L(zEEf28cMeY23l4<}Mt{&#WtJCC;ayOtX&*1LCSREDwOMg{t*g;Z+b z>Cr5y-3!rmH(`>w|I6A$RyXbFO*BvI@l?Cj?DKHhs7UclEo~Q0rg(pL)x}KS-hA&I z&i1sp;6uTyG?5E|NP2j9tD)7{&XIzmigd8KM)K6n-g^?XK%IqPHquD0#Gr;E@N>{b z=m;~-E(#jE4EkE%US#;o&{r`ohRkxfqw}%eWGOgtqF(St<%0C(TCbdA?eOioM7BlCz0LrqYMh=TI3kjIf37CL`y^GB z-3;B;h2NEx|J_-2pT4s{Qde@N;I5^pvycOanv16Tc43sm85zy?1Gp_O^_#xbOX5>y zYp}BOOGj*$W~MKzrJlpAb&nY|Evp80F86^lHIc(FgY63c5UpMDq9Hma6i6gzM#Jh$ z?Og=h3c`E7G$#G^`U{e-*LzEmD3#|;;H2$7_Ge}T2yLil7s8P!*HT#&S8!RAUR~aX z-_yJCx&A8l0xn*jnTQ%t#xJ?EEtTf9zZ~6H#7v1ww>0s@Crr8TvRPq;mXxr=gq+G} zklpS;ABl>QICVZ+`Wa(Ud<=dBS+Swe!f%k64iz^ZLgndoxyc`_kQYDO{LRJ zYzsYY*F$x$4k0BCokeP~9I+I3^vk>1dliRoueU_{0g5v<_#>+?eI-u8*ugRx1V2fA zH7+b`YDeNJM)||*u5gs`ccx{bE@5tAN_#|*MB=ng`7!?A(C#qIQZY`VN(ZwbSDc}a zs=nqkIR6IUe!|eW?Hb6Jjl8FzIe!y{U&5Usp*@t`7bO7s)a>L}i1yR= zY`;34y!WE^VQW@Y*BJcWz?mV-A8FK{Ov$vB-S)2W(9ze9xmj4z6rHW?XY4i4L7pGF zeJK>mJR7IDjfND$hWUcpfJu6>IPxGvjLtzD(>I zk~;iTrd59BYUSH#NmZW@v2%el!s580MVzg;Zj= zDh3k+Rhvw_=`L~_$`}SWo_HD3py_dG_F!`Rf##miK@LvT>sJwVZ#Gpgh+pg#*F#Xx zZPw*hkP1B)zN1)py3IyMBKZ*gjG(VN2&|oGVJF<4p2P%#Q*F&^Ehbh*r{u=J|Coj3 zmmA(EDr)3@C%8+`Z_yZKPg)wik(K3C>u@2DW>(gHDyc$UYG`<8dfVB&ulYgc!=|;u z1!5PQ|K?aO>x~#Xq2aY_7nn0cUa9WqZO)xn@mt9qB@EKvauyYpgBS0&R{RJU%GPqE zx18ObEv-Un2gBXsc-m4z83Q&G#$YM!a|^%2(GUDR@26X9dB`PHq=V|EhrX$az}+&(il}rk&B64i za}QXz4yf2HRfTt0N7`&h^C_L26p{z#gShw3md%Wx<-IWRc(j0-&|}v(b%# zVI~TE{PQt$u>CaIrx*RURHPCmV9%%Dv)dkqM(D)n{y4j4o*L1F0rK9ltC>)?4C#s6|jk!A* zP;lS+=ndPtXFOd;T^W4**0#NOl+R#S$=)*|V>t6&(~JJDQ^`|QunmK%6T+8JOGWiC zC)m$V08(AxvvZe~z+gAJRhjI(dfK&BuKxy=EjanH`*$7@V5oeVEYU{GuCT@&WC3<0 zo8a7m%F2pL?mN8SZg!Jc*kv-=Qd66tt)#L2?<%r^B!+^Mo1<<$5CX!BgOknvdj_Ym ztfeXbW49Ym-+Kj5cR=1`K3I3H@8fUxf^RSd14+O9I+Ej;+=pzpJz)m4M?hg)62&vu zyCt}u#!}5AGPPp{&;LWzel{u^ zoGX1iMESFlvwtz*{d}(p`4|(s-FoVkI|IkSg(y^ovd%}2Al?BBg888NTM?4 ztqnHG^mqp@oXQD5~oWT6-JxTW~3xl4)p*-0m{bJbO{}86ZZ>%sqz^) z#&2HZ)a=)uh9oHQ@n^4=h|FiY!Gmk*RhQjylS{Z^!*CsTZSa=0hO~4dhlgR)`X1^c znd<#LEX)r@^z6SW1jwbegsY3@FrO{ zNpkn-tc=7wj3rDtliCt)xrk7romcMGt!@qORjJ?{A?}t*bxEyZ-RhOItHoQdr(=x= zU8;4@_9qx_$~$JSJU0HCGzYOvb$pk!<{!da=Tp1Mxw*wSq!w;@QQm0l^-afYG{U=> zAjGHh^e-LlEol6#1p_Hdc+~Xrb>P+Wc-I;RuSdPTzdg{_V1U12icy6!ny+fbaecp; zSizSuHnWb<2HR416bq0G^Wyp$BtNF{h|0g0<>NS|h~YGfk9QJ+_LI+=c%mZgLoHf_Fbe)7QyR$sZBUOHhyXaRi9wrj5W~tBjbTxA4MMt znPBzEbvatxDa$u@^#1f4TyP>L_UYmdt2NcmmY+gIB6WN=lgOdBS)9R<%MsL?*?Zr# zgT4ONukTuo5n#~GVo|l=hQqU4P?#GHbfL|AWnGpLDvgqKch~3MATlSPsPodeaLdv- z!F+P4fQnBVcEn32J zimABvwz7F8b+J?KX4BNmnP4ArK5z5v`IoY?M_oe3U$$8k-1tP@Pcb`}hVa{XzQYE+rrEyc3?(WaWoY%4_I>|G`UIc^H z^9QCTdVZEt>FcSDXJ*^?Vck)xn;6{GnWL@DrB!OBM>s|GYK30!R^g76jgx~h!}*j!63H6syvsYd+o)lL!;u6*y%DWlPb-^L7RL0+`* z?;Da{+xUZGjqBz<0Hj)Q2CFay0!QpMm3yCj=Ep=1gMcQb|0?3hyi+Q1`S^6Wzdg2rd zg8qOWb&gnAzosOJTvNedE;0SRJ#mTq3oFlf6r)s#K+O+f#7jJvlga{4Fj8d@oW6o+ z0r9c9$-5g5!Wby7F5Jc|#Rq;x1JmZHM(mC8ska`+K3kXMP22K^H%wdY(*@vg@YLW- z^`&>FTw40F%UltAu%W^x35Q5O*v#2~KA&>lDB+9kM%ZB2(rf%v!O;M)f@uB+Wv zRh5Ukpw#_FQ$j{i!N7yXni<6&a&46bXOw#Y|S z)=}fn&SOfX2&LCMa0c-Z!Q0x=LG3`<2do7J5xD(V@&+6^5d%T9#$t4q95nZSbWQ#yCeQhuC^9Sp2z zD?U6<3kIVKX4l%eO+*Llj0X{Ls2L8*O$=(eY#C#<3uj1hT(VX&D!;;gi!%y5)6Ejm zLc|bXDJ%EX=0?8TwwTFxo&P7>@V12m>{B}bRH2$<9#@*I2Ct+>Aq=mzXD)xj^?7$C z)vWY^&#(D(_OV;COgDwV4?<@*dqD3Dsj-V!8NZ|laD`zqJCdd@zI*VhScwlheONd9E zU}1P#zdWM1efH)A(aNV!Ai85zE`jQ&W7ze@ACCLyO?d;n#t==#r2X256V6Lbo15~7 zk>Gu%fscVBUvX+Q;A)knJ*Dc8`Hz^Hkj;GKI@S98ew1Q}FFy_<3SF+qUiiKYx!wL! zl#Vvj2R-9i@7|;mtUdMUi^@}B)!(1)1wY>Jl&yK?wL&z@R2C)^^YVi^lQToK35Jvg zdk5d{<=Q-$18)Z9M3)VGiMObH`%{+Wx^p;c8Fw6ZjOZqkjyS#4iz);t&p49_VzG#K zz;G+5?c<}7y&#gq*i`=I-;GIc&!E!U`GyF+jWC!xXEB5!_x;(4mh`rKiEG~@z}>gv zZVKmbkgZcfVUCY>HOnT+Aw9X??z%(KW(5o4Q&ZNgyvM9xbc}|SA&hnP9M9GWi%*Rm z!=u0Q@G0F*G$TD7h>9=%b+~6LyrvZM`U!H&8e)G-v%ecL>fBU=Arg0yNUubqcto2o)6zleU`s)p18)JsS>hcPifTy;^v zQR<%5i>?-aEtbC8}{4x-p6O5XYbL|(KLL)L|lnZ4|!f% zO7I}vJ(3Qub$^-TAI?mB3rhRp`5T;r_jqVFe>Pj?<~d#St6O?FJGTf)25(OplL!+g zZ)8Z4oRS2Kq(0+%mM@0G`1_s2o&Oq5n_<`StyYyIZnZpRzlU9W?NB z=!_7~;{ZPxZT=5+&h33K8hRiPz3P;0pW0uc5AM+v<{?gw$nM8ysAeIqFnlP{(g+2z z`{!;Gd6^;qcORlQ_x#;M*Lf1TFINNp+q~Wn_`6EhP2`1aPz`|WOG)bHj$~@8W;=rS z0T#|CMDV~xPe_zIu(!mkB`vrmSvQRS=oZ5?`Z#2%bHvp(3Qdsz!(8z!Wl|`PImN*L zxy^V`Oumb3?xeo=cSRGIf4(58E#R#8Yrh(NT|kugqb($g7jh%+USw%S|JFqAex>;! zSIMw4Nd{L=ZL&@|`m3|Lfmv5~=F%d`-oBP^dd1<JEUTimPZ+n%gj*8NF+Tv%!iNBff3taJa9;?0os+?Rn5_ z{!g-i75iEq$f{PWghg%peAxxn0&Fb5Jm2pfkQ8XGu>TJWzMfu zMf8$P@Asz%JB~pD&Y8PBs-pCbyED7h3$r;c(|Rel#J7Zsb}DfdS>yRX7&4KaK|g6e z{dD;?*>iXM<_IAcSpv74wKitWB%l6BoVw*Xc`F1R7CyUx9nu@{{NyNd@~Faghl;iF zK&QbhnRC~;MC1P`I0{bPKqO_88f7!us^KiUzIu4zo!#Fr)6?r{%`j2$#Tui}TYb{^ zg|7C#4D>V76y6+2G$_t>o>g4QIecDjpD9gxb(;E)#6*BJw~ET!xJtbK2Sn>G9sq@qcK#%BVV;APd0>8reF+RV^R?LvyaC?m0s)uGQr$fb=GNYtzyC*FNd#ZId$3RM2p=JQ@DUwO6B2 zqTNNPZ98Q$ES zzqegKsMTk7Y-JM@k48@W9bJ-Ph{u}XjQzC>-@Ziqkh5^3yr8OlAyEtvUn!b7I`(fl6$rgZfc&vtz|x6r5T z8XYGQhoARJvE6Ai0uguYab?TF19T3eSrPSj`JSFW-BPmEn)n7o2pIyg6pE$@1Fcm5 zhaxH%dfm1aE}M$bmaBeNO*P2Gh=xWx><4QL%FTyR?f#LG^v}LHecv^w`n9Lpbj8-z zJV${p8D$X^JBu(C%{n{oTg=Oyo~egD6a$SEnAh7f*Uw^ly_A*OsmDJ3Iw`t}gEVe^ zwLTLMcYKd~s&1L73e}9Qh~xKm#u(%} zOc;F@FxqZBPU)Nm)m4Lfy0R07cOjvVa^FsQ#bCaCyzONFE3;KKibN@D)K zwHV}JhTUKxsLP!p*~R^%AewGUCiX2@J-zoz}x~kfcM|yoTCyVc+Es;#- zh_tjfN06!#K^9&xq`rduT)4@Lu#cWjyqs4@7OAr0<7*~aHT0+0@&^+6)@G4u6m?j_ z!NRxHbvGKnI$kzJOuiEqUFtqrU$Mz#>;#If5jQ!O?7H=pF=KbP1{3`kZ=6Wqhu5;aD$3I#Udj6#H_Ib5 ze70-cbm8pYXKG0zJf`el!`ABVVj|Bz<_TtbBhsi8cQ^P7Zby9MdhZ%P7iyBV;`pKV^pj7YbdxR%;AOUsJh!T9 z_{3n&9G3R-Z%GD4!(@q2nf*DIs(1QCiRv!;c`1V%ymo&h&sY>5z7kP?zHTXU^P$~T zBP+bV?zQ{^&szwat+XU@!L1A%`MURfy>;SwJLqjF@x-<>iLD|~^K@LKT)%%Ycac6s zDS4|aP15!JMl1~;l{k-jUTh{D;|Cp{+0l~7%}4WmOFyzvuvCVFwQ=7{oeZzvk)#-_ zx1YdmFq;nSa5@S!fk>#3BNm@8Hry{A4KeP}jm0u4k({2BwRUOhBoxf|w|@-O@1-1R z2?SfezP_<yrxEf04l!^wtrA|{{6rW0fRqeD z5J&swKidZ(QYvOJoeu{OIE0{+kj8bZdX_QGDXM4n+gnMBr^0iS&m1bnV^3JUYtA7- z`resoUrO3<(QCHFfXor()c7QL4VhL`<6QZ86a0ts&}M&eSzEhB#8iB8cLU<`W?+WN z)w#dkcfWWeE;H);-N8UQ-{bRF|86gc%e^LTpz;*$EER*Ne8D^9czcTPBVxv4f1$Rz zj?x41=WMdVjE0Dt5f8Gp``>c^>fN3KnCG?5uG$VezGwvZf<$qw3gp8=7N*9w;kDBa z8Ol=GRX>P%+qK}zX6)i+jW7BhjTGrhJDwlm9ey)}>mR#6kSeB(K4197#Hw28IwkW6 zCds>HhamRIo{r=-$)VfW%hWxS7!Oy8ib>Y%U21lw|;&p1@Y28X?OJe)&Iy_}Yom<5i6T3NdeTU;a zYt5c^Ww(AbDL!pJEheV$ax*q>yBg6u0F^ED7Da&yMGZyKMR2DL38}Yv&q_+#G_bj7 zA+yPATNT}!H5KhG=M(**L&TG8)+rzppfOI|<>%_0TlMs`eJ$o8R{wNmSQIQPp3mEC7`{}@lwrGhk<&){Ze zwg`n&O5AHS8@iyLABCC6W~F>R`zrMyd$Gx-KwYTCwk1TlU>}w;U02RujT9u^O;>Zv z5gnh1NR9CY5xu6^7&$x+Q}@a)%?q?KQ`|y#S2{kYn;xy`+75rsM5@*J?NJNn!h6yc zI!li~9Do0(Mjn@@$Intm%YLERNabfIkXE9owHp~5p|n+EhKla0u%0`VyF3Hx`SUsw zed1~=#YPwnSwQec+qGT;Y&8r~Gck)0-g|%OIb%*(9xSJJ`28iIK}_^`yuIN{6k8}e z;TldSLDt#0;1IZMhe&40{|(rB83rBo+gSLUU9@m&h;U{hk`I=T_sGV$vQ%`V76L1Q zKo|vAw4{s_wv*eN$&tQBXMF;MWB^OQ8XGJ5zl}#eyr7)zkN#J;m*nEq2E%V`WTtWZ z>45__JJ~d!#_msdNqEu?lI$#mEbSiKy6FD?1In0tmA8j`eMD@$cI-C)T?Jdo?QBuZyMJGww&n1UV+Udv#tpx)Svei=ak1@+6t(i@9n;mk!vcAK> z9eI{N*Xj*3YrR|^R6Cw{@6bQ5KxJBt{s{g|?r&+^L1w;R@je{_F{;l8JhYe<)u4j( z4`VdlTXXx4TNDw}X2~7%U|ZH|2y{ehF9pzTDteqAwN$hivc4Yb$CQL4p*rLvHpYK5 znPbvI7Z(fpdkhvNZC*MqF1r`2=3`)KbXZTJOlb z*RU#?sYA}7NQyR~g5qVc!iA3Xvt{kt2{+Qe(i}7c6Kj#YK2r3tG2xhsNExYpAJRx@ z_?!D73hrtGqC58q#{7D(-K5m!On~_Mu=6<1hF#QnL9KGV6QWYB;kWeu=(2iQ#|MBg zsjs`#+*%{K**6|K%dfN|7rYI$3|c*vh0*3s`HQ{7;x#AoN?ye^EE*0E3TYiN+}Y^N z1AhOEgkDGB$v(2~t%f(c{hy$)R9|wYo|ir1j#&gjYnq~?hx`cK`C`QMrIwv7SL8%C zPa7!SuAeAw`iIOgRMVjmZ4_zV*Lonm_`WFH`kFO-gwQcbjYqye@~6uhuP*d%lfr}+ zmh4M1U7bI_S_47DO?+cQMoA%kQ&QM>vPO(ZZmT#uTJ$fZam!r!PKtY22HisIh@Y?! z?o13AK@vB!blk0?r%{53gVdyN^ZYO3072Lsk7YU-ud5cVRJuKhTIuEc$^C@9i1Wzo z*5l|eq&E~%d{&eFZ6x%z`@M)~IT}BIKiVhHd4m_%`J&fpSIm>p1Kq=f!VQ^!ICBiq z^Fe_X#zGAo#S~T)UEGR-A|(o6M4?>q`Dq&+W5HAxJmxeOCaS1sRoGisaA437`WeXr zZPitBoj|Nvl9|#@CM3JKqcv@`FkA6{A$fUBKG)Gj@?=M3M_HbvMD>_TiesKGi9Yu& z(}iU`EojQTyhJX_-y5gG^__(dDQPpdUGP%- z@fo6DV`KYtRRh6js8RP<2T09c;vw%aH?(-LVXJ|oRfgWZZ_d#tv=cLuUNp2tLw|3+ znzc*~l^{nCF3gGL6Ymnu+_#me$Ix+o7*K}`aSulqs(Rk{x;n}pm{M&KMEee3Eb#1C zOi(Uce}gHf+C|$Py;_OYXtFV#6X8?=&b}lrbc8-)B55q$I*(nlI

F7N zp(M(`qC!ksm>KdL!OAf7Hr${ARoRNq&wHFl?(?0UC5aY;FwvJ+kEYe-WVSuJDbWOd zCU4yQdaVRYEJk)I6B#!TU1TIS=f}!hKcSDFf z{&oYd31!^tLJAdh*&#zN^_D2Nf-$hq^ z>e7q~oeB=sZxba+mt+CagBr65lnl5<;6QT}QS?~HV;)`%aZ6GZQSn{155?oZH8{_6 zRdgF0>WvM>*DSeT-w2RKx4hHCe3+m`gn_NUq%`-9=(8K(UUaB*f{OI1Iq13yo%^EA zjf_}ivV+G`SV}~{Q@ie?j<2i)dcLt#?G66ZQ7=`SyWESW_WppvotvvsSIg`a$t0Vv zZ@WftBmpRjJL@2Fd8CtT*woG+4cS?ZLC|5D$ObOUidmA;i}f@tNja((_DiiUt`Qp} z@#}q(f3f!iNZt4|Mt!Yqf4(cff72rHsLKP7Tv8J@-ti5>;;%A&G4xl8HW(W-(y>52 zv!Na+9FL|?L`(++X-HzS2}`MK)-@!?*5s74V*9#tG(3tw6&m>bzjp(C1VBISW@pHf zb%OM~{Bu4C8RWlmmS;A8@`~-$(gsh$8ePp#f0e$pl{m|zWZzj&Ey&el40M%$@=zP_n2?l9!MgU zgK^1{3qZ8l=FD)f{SQp(IM?7*MJ6>hp_38=7GvH*tUR%M>2psD?m+!X7fHEO8{5e?$tmB&Xr4EP+J14m2w)%b3^jn={Nl#N9elN!}-4 zb~D%EknjSV?(frP3q9$qJ^7_G!)PAKug z!%xbLuZ`7Ar=`kT-Vg5E;8x<|)Ckb%RaFF9ivGko=UYV9pNkmK9J#r(b3N-kSnOGw z!iBL|ic=xpS(7CdJtdF~uhI`J3>ABe7r@u~udCYkJCX{4+9YUg2%uYvm0(LMaP`xS z{$M7X@BknvSb6On$F|m@-Gu5$Tp+Fq;bOjt9}8D6YWC(625%7cKDazFdVgcG@&PZ* z=_d)J#zT|3YN^T7H_KO~u|q2eq~f*5a>5eOr`{!<4*b!3p=*+_NDnjFmFwhycE&j( zMdh8iyYL!mK>S)DN-9~2lpRHUJHHlcO;sUMOrY*+Lw=m)I0ns}X` zXAYDI`wM(-=BJCFcs|WKC*RVDmxwpB?*y;)uIce>VggKHRLHY@C^lrCA)+bJv~nhj>H{c$&bcnRCAw}@H;{C;UOE%oVHHE6lyCJ-M^9e40V3Me4sP31e z#wF4le6Abn{qQ0wv~1@U`rRi+-kZ*fHiiV|ma<9NcM|Gmn*}GaX7Q_;{b;*|X2S;D z=iOs0nNzM%*d|!gb)hz)<10x7WbCL712g#%LPu!=y`h)A z-AYRO+RWpm0&O>jOTO`c5_&kV@H;j;^!isJR+F{u{`kTX99Pzg8Rm;vj!e*1S++Bp9&OzwKxy@7`LsK9YI zs+n(Dw>>@M6rYurt)zb5KKn8M3HQ}Nj@pJR>_Q#)d1fI9Cb2WHEy$>Dm8O)V5t7mo z26q7mv+ZG)s`j2HoIn`woAj@18YPAx^Y#MCDgtpnEv2bwbpkWn5k2V$ih1obdnH~} zQ`o)3lyp=`zq>qK-^GcfTW8kYf1ai;k4h)9QN20WHK-}b)gc=!dV-%px z;89Wu&+aKrq&qCmrz_!VW}|a8Pk&@9EDj)c)GBV{LKTl?it_4dgV`J~=U`oSVMY_B z+%4mAQHIf+XF{Znj1y6QsXc4MuR~5iL)#{z1Ul$560Tz+!ffe`9g`_u=nh zy5^RGQI&z&jNBnqlw@WHK2o?}aE z4O~k*jr(zU&Ee8PW&CQv$&1|$VC2`U%}dKJM}poh2UyPQ{{n!-!shBL=0cIhY?y`C zdoSROjzUl0-)827y+t+CA1!5mN}>^ZZ*JJSe<|-9{n@6syRcCm2Z|rB!x?)rexcI+ z_%^J=8%Oon$Jd{z-#OfG-0<*V$EdB7p&}D@n_otXoxkumJswZFuRx*ux@RvHP2i~g zGs`x#B?u6)g^VJzko(|TRK|@gA&FCz)bLoXqj*6Kl}FWAI}+i^+w@bero_53akq#J zJ{Djtz)FHpFxASWa8s z$-rg$e~&3DLP2k5zJan}Xl!r1w5}VAB!`rqAp$n=tfyzVFIJoh{{B9NALSFTsk5`w zNI%T$-~VhsTTj)KIzT-lG>0HpJ^PAf_iqx$$)HIW`j;OK8I(2E-IvRhY8A`0jY}gG zRB1_jrlWDdP??-{LpuaugpVv9oFU#2h$VQS>kSgCv=EZZ!jp_M1zuKtMkOv)?mzbV zM2i@0-{xo6fAqWeA{g{PA^3hMZ!c>_VXC15X6BZqd4ANZ%RHNNjFVqJ1?S-~y!S)W zhkk)<8a@l~mNn1&rkvU9>qQ@2!VzW&w|htRv>p`jkJ+JLr{i@qC-Nr3nKkjAB!A;H zJx6W7W?YU`P1^8GY3W1&sVUvw11b>Uc4{B`wJzk_fn);z;vcg37SAiV88$}=?9{fO zD_xA$O(6~|tR#alnF49p_m!2HKa`vHZ&$wZJwKvpYu>z`?-NymX73?IegS|zf*~Vd zP*duLd;}YTBQQ_-WV+Pxzp2!gvoQeNjcMRij6aNFk*E;`I*}F2M=Kha<}zOqkJRrh zZ%Xcl6l)|2sHojvDg?JLZa8X`yt1dyttKHyqHJ)lM|LUMeVs6U5{;1KguJ6 zLLc*$Ph5o#-jf_GmMUSofwEM__x?!;fcTE=u-!fa4Rt_Vo}cdwJNDM#81zS6nLVE3 zM_IMiwC?3mE5#`XEO>hJ;CVq+YK{i4@ z(GEQ(I-jJ84%kDgh_Sg!Q_D?ScdRz`ZvrUXR?Ah{P$i~e5Ys@&UMg_I7d4+Abe?~A z^L_e6AwgRITm8i3arX=uWD7LQu!}<5(TNIgo*VYgylh6rl!4W>hMHbe+{}Grj{N}H zFJ>2t^?||mNKfxO2QZ6H{Q&1}VRi!p0oJCfI$jTs|I)Oa`z(%SU&qiBPjDcEOi+Y- z{$C5w-Tz#B_w#D^$V75PibkmVRvu%%{`3;&`7rXG-p3xRB}6{+wZLZ3ur9W-D`KPE zfmtnm%BvTOnT{q#D?U<{M#j!XqDXD+lvKFsmA)Ib>r#Ft{%WcNGYL#V<+;QG5w6JrKvK33%XR{=7R7bC3% z3M6|^kAe>5MF6uV0m40Am|o@4;0Kp8Zgi%i&KOu9x?szB>KseWK#Mp&* zD^j|>8Hjo}d5W!8yRa-U)#GQ&MXfP{9+s&>N`xBgV?vx)4nq0LBuFxDrnL-*=)4b5 z5qz}sW^PLWMBu=GaIsZ-OSv*UZ6%tr-ob=3#G96wsnQ>O&mUI)W^m)uQ6sPKVtdG^ zcn-+%PAUh?nDZqyFAa7E593%8`3u)_hC$wrEV?mn`1Sd*Ah0?bhlPG7-Mv zU;qT1MX^Fk-vzNyewc1~9Z#aZJm?r~a2+LRQG6nYCf7>uPvHC6&0qZ8HG`?ixBSEm znev=FF$R+BEQD0TMM{<<6%m0hZ~KVd3S38b4hX6rsq<#740Xs!*K@P^kPnZ)o@7dC zUXU+~>uzjeQr>^C5`GltOW-5Lj&D$8(3VjBR)Te(`d7OOMCrC$yuzxrsL^0p7NdAi z_V&1JKIsukBv;XEFf6Z`*8ialkxpJ{8FO9qar&^kHp;($MFg!rdBX{|q(24TGU4QJ zBotuL@hkIy7eqT6*u`3G;VbN5?ZGjm-orSXp>B1nUDFBp604Q#L1xXHXAAfGK@syp zak?D>gy@LZq=%G0cXh1G>&DJ%m+>o!YH90wECPKCl4P|KlOC(Qqsf@`#VtxQRe3%S zyQ?7ZEBQzNiwOlk?tik>GhUBYssAYdv6l&ViE=`?V!0R7!rV}0VJZ+ee(IvEaX%>% ztcNFer=Smvee^pSlq4OG{V{r^lbRzm;v5nFav5^9QsJ+4hIbyfg1Ab3<3AgF0jvDE zrXXt5v0@x*RrvMyjFtLB$N|j-8NC!=x#}JRml_vq#@D5TodRVR3psLe%>|IAgFo3? zc*>tCFwgHgJe}1E*~vtGOzYi^;or1J74#pMiPgH2dzXARh6Pt`83Oqi+#jxS$ddoC{+mzG3uF3>Lq4qC+;w0za^gTD zKAb{hMjKzTzt2a)z(t0%6%vdPKWkKaHi3k-I9W@GoIUA1It|_}D){9u^}%&)@wb#) zrbfCj(zXb2nE}W|s&2m1FWzGW(;A4vk;`6E@|iPDfdb~#pm#1@eX9Oo=yrlg-HVq@ zk1}xIAK7db8M`Vc_Xq|sDBc?Z9zm&tXCbw3$A3ICy+R-UQa188PB>XIG+IrrjXNnW z=_-BY1vJtC1Z#2G?vR1cbR5|%=Bm-EL`O&e#hMBrbAjF*ZG@D_NsWmG{HZdVm5XoS zBP`JKa?(0FnQ$jdi@N!9mj7kPy6A-luR99wh%UD3E>%Pbx;(uja2o6q}zTAr6hhN+)A-w2MGEU)9j2W@k=*su7v0 z@swd3xSE(kGI|dQ(R|!7K$iv&(e2-^G#TxHdkVr?{q`1nTSo{)>7VNKv;^$S4{`iAHq!y$WuGG=&Gd z<$r(noRSiJ;Rxy4B5euyMim^x%~C*tU&ppI)7nf( z43?uNWdiFn6!1v;U%K3+%NcE85^|Q# z;J$u+%`Vj>3nNvlCkWAX-&Zd~FJV^-p;JUJEh)*#$??6qy5c+}To~eyJ8WK4%)Rabk8e@GQss;_K6l$0fv-p-Wd^ zn_}G`1y9`ECE3P6e4nLtD98~AA#`ZKu>$yvL8jiyf!?fZ1=82WtY43(Xv-DO8HjQK z&D*nGXmYo{D4%9z8#=fm+t=yTVth{bAKbw88fU8H*;u zZ6r!3D`PhMSM=PXtw<)Eiy7h(?Y7;OF~(2leIPCf!3)W7E?;EVcpgh5GY%E`>kpsH zjtD(2FtjikKpEb#`rl5_p4UBsm?&%EeP&yAUN@kpWGRuavVGzy5#eE9HvjD>jTb@@ zndGp~G08kh&z*KVc40l6Xl5ZwiSd>(n9`VX3*{Xyn9`_{MrH@D?^Kkh^GIMv&uaH7 znq}%YPN;yQ*S;Eey!eqjA&S#!bjWExCmPrZ$xTEe`3kTy9>0j`ws^ZwAC<6K zIU*qNZferbbsjbTq^m{f`Bg=9H3}KohJ6>)8n;HK$_y*FiT>@fwL-lBAFR9fP&79_ zjk80Y0PkEiE>Wt4lC?ikW7)Shu34H;g(ff`J9MF|vT(;7N7XCiz*C`dw7y7N)`fP<$cSCnd zgJN!|+eLc{A{@w!8}gc&sYAs`fe|0GtJ*GxaS=1LD;RS~@KBJ5-*vO#e5+f6p|zJ7 zoQ1fG=LCh)ao~}KZ_XKpRJ{Koh!o~+`FsS7=HK_K0THJ~P^g*S%2nZC{6 zPCp<<2+vQH484yy&uYo)0ni!{LZ{+wL}K=v41UIyxCA+P5d!JyC4sDQz|@V0nlzt+ z@S5HA@U#2~v;)Hks}R{;EvQNCv#{G6&j;hOL))_1)Hj@8igqg$ug~*XV5djriH@s` zoc|Dyxn%FR1q7*ZXoOxl?;y`n#QOX?@uuv5+CD8H1@w%2@vr%+bZ)k{_TCfpM=o=Q zDcE~sPM5Yv3yqR)7ScW!FPD0b78G0j>}drkE+x3IRR5eWLuU5FN*l&{Hw8sR6%N}W z%7{I2Er%PNCk&aYSWtg)lZnx}q&L{%GB~}XilgpLW&+Ll7Rpc^*l-JjA*&&Q#RT?r z_{3$bVtNJq;9zivQJnnLNUDvHgb@OYvkw5uStdhWC%L#$&_$r}P|H+eykB$UCHeFG z+7^y2kY(A$4#sTkjcNCAZ+yKNS~*TxbY7wB)&{hR_H@^FRnrrt^nxY(cR$(a00H@N ze=;^VBI@ot3giVL+TfP=xUqu-O&+WeJ|pE(LaZc}`0Y$J&pJ1GiZ~j1*E2xE+*};) zfp`@CMA0?fr|d}6mbTuYyK|X(T;X+AOSim9*jo(C@CD*sW?_Q7sJ}SR>s`2Vs2E+8 z4d_{tvOGl$l#CtR(oW=JMBn8VnRj zQ}J;_lSOIHeVd$_#o>?HcTd*$c(tQThLtfG8ltO@f=BJpCV)6fCh*Ax`u*{}ZW%bw zTNMS#l4%f)Hw|?i2(N%=;olTkEsQqwY*+Sezjq}TU>=m4eSVE!vvN_QadVcypPGz8 zC@bW1a5GTl)s9ZyD^k8)kC+Qy6h$L_J`@HwX*5mR;M#2{2}oG_630jR$M6whRV9*u zg5zJ-@8spjFS+TM3aU1^o{=W}8aI+9lb3hsis%&K8?tNd;H*oc{i-=ho#=y^d|-As z0%d(?va8`S8iS#S*;}T00xpouGZT6Hf|Ni&+(Fv0@O(+=n(S5h zy6pHzx%`}Gfd}X%t%1_z%OYa*bh@lQuFVx4sxP}w1oPQb;;T4*?sL#;U~J3?bt@5c zJ)iKAQi1fr&X+qNkF10fVDcEXy(XTIgXz2&SFW$DoLNViB)KAaNUp_0j$55e6X$fG4ZJ{g(;rFWga>RUPd!F-!LI`K-C?i(L(fxb&bD7PewU zuMaf5!f;fS@be-w-Oz?~9z{B{Fe>}jhq8+?`}fY=U%R#^dePLXTEhYOP4_bfTs*wr z=H|aP+ub5?*n;H+p92ejCOeOL(CakETy}jD>g(&z&d##RAq zU)G4ml_MYX=e}t?A{Nx*#*g~0%yTSV56nK&JNxGJlvqDcZ+DYh6x0mJQCG$8=4;bG zj2Ef7zZbSna#npi z!L?;~JWZmcq-5vpOv%BKSWr;lpq~@I?zE&^q2JZHRz)?(a~}G_a~*oFXTYUuq=-|q zj4ze?al-X*K z_8%Z)U&WQjLOS(b+p{s#9Awp7eg3BY*p`-kcIZ``9gHSEJw09hH`q;&OM^x&dj|)jGc}sBypHp|p=c^Dt|sjx>bZ%h zyAZN+@+NB+PbR$b+LFQ`<8ocG{MLr8`_A9B$XU@Av!Y-bR#zc)2sECx5L%j`NL9N( zWR4wXy-1$E{D?nEZsYqviXwAvUxKd(Y5pEa(o?Un&bNbF%y}+_Hr060xRAlE=B2^gBSq<_IO1MF{n22*q1I4dbUT&!KB%64Z{*afa;%Pa*?6zoB($)@- zj!v*{Hy-)c`F8Vj+?0)if};K9w46Fo8ZZ_&V3BFo`~nvZk(RP1uueXKaD7bH8;h0} zd<-8@30SC?TDC?iwd$cLu+;+ju5KawS{TK96r#DvNJYZwcc@uxdfvg<-CKrMf!APS z<{_QCNDY>%{suOpGS$o{OR}ngt$}Q_Cc&}A-pUe4)t^I-shd3m15hFPl@w0GHv z1t&$b};h2&Bw=*0%uJL0N)A5v^FhF?5kcinhVai72^TtwI zTFQ|0w$m4aBjCFqVz02I*f~2P5JQW`Dk!70G3ecKbm{$;lN}R8%DTc=h;0wLflpMBs5njgQNa zl9Jll+wUZ*>yc1W2KNNOIPCVrVKQhzPVYRd*|bF}=1bUvmRbA*0yNuQ?W-rWiy$Du z+w-XrcCK>I+iS(g%Hv*;zWNc<%5NK{Wt6~te`5S^kOqasu$eB$=cf3Zo#yzQ5za1l_aS}yO*}L>{)8?CM}jU})>D>#K7(MY-y5{G|^dG@aj$&Au;SiaW}c(;7zt3kyk4l^W;f#Y|$ z7Uf0uEVyYsjAO;N@7hP^D;M^zuX#nc4z6L@UT@uw2A<{g%QxemhKy_uH=0?2R|+l$ z3+M|+ojL6e{cW#!NXqMkfCB3tDUbStIxMg9WJp)o&TPnq3tpclX+ui`Rus#B70UT7 ze}Sd=bRrOLz?_zCt=S+)bJ>I|A|hg3lR9A=&EdK?s}0dV1Qid|7d$l7R7vRucRZ;* z^!=BvBp;MWvp!0omp?J+tmI(nIqw(?-uJ#AZrZIo1}f$OaUQ(^UwO^kDDBkuP81Zu zxm1HO+RF&MEx0!vmU`!s3K_>B65gDb(dieU&9MjyC!BxG;^gPqB=;dtH#Gg(F>fBV zH+k5uf+BhdXR(<+J#IkOpG-W%O}O8E!yyIq)lOX@!P3HEVX0;7$lee=9Zt@redhgo z$^pDJM`VV;e7}HoUMm^P#Aju2QMvdX`_|NM@Krh3O>#RcqF0|-brHjU^-b+2ucbOw z%<_xZ>VZ=iT9-eR!drjsS(xfcO5VxlGgyyzRmJ&1j%Mhybm+-LDj^|Jr2LjfdLcch z-CCK6)v_jB>bGt1zt62zWSR5bf~Xf)v^vfraUuZNXsHgSQGcEOLtdi2tUws=M70Jk z{xLKvpMYOKFX$ zml%-hAcBvF)0P@JA^3pUxp2A~7TH`9>ScX|=@zP;crIpeUcn%nV~gZep~didUE3NS z9{#mdrQ81b@rutg8_BS2+=`_xRGO~z`=upX0M8*)rLQbxgV`>Y0!mCV;*$WhC`IVqCcXT_Ro6>5Of>@a_W>AmZ6 zcK;#IxeA=Q3npB>gocGBX<$HdeSN*bXO?;)f+7^+Bz_t%HqjFGuqUZ*sKXwf&LFeDx z)SKut}wU?c*Y>kSLjinz}s}(TRm3+{rn~`TamqAfH|)9ptN4 znX||Q=^!VU{%)F15WnCjvi9*lrldj5lrah>ux8$8(fFqik;`s~z=hf6Vm&LpU_5At zOW2)3V5%`@mDKJm^S(`x0YuD@qvi%l}-g1=X@i#ntFO&)>Cx0Hzfa$FDwfl%=aT>_EBgyHV+!{gU*0sxPa&sApP`;FP>JT2rjAZV4 zGlR>uMr-_|^|8lI?PK78#Pkdi8m$CI|TjqrGASX}}c*)q`%W@H8^yEfyW zQj?K6*tC=OnEJ2wRuOD@X3&GMg5@gndQOFf?7s}^H<4=5GrU^J%3^wWRHcBd*z0@% z7hOc>%K2N|-5XTDYsQN+saR73SEuUpz+HcDlCq!D4Eo78(2tKR3kn%4a*8{ z?s){&Wz74Krw#9N0JbZYOApPXuAwM_F`;1vxJfH01;^Kb%_^~bT^P=3Oqh17a z6$aS`WpTqNR8G)>0r|`WyRO zRGxj+?#~|!8q%b=xZaohvz10m^(Gzd!*6>l1H&nY*v?*;Eru`cOuYkAY5p^$6lA|) z5ORPb9iR$icK1Y+1r%9YN*s3l4d&rV!(?*e%tP87^pAL9$l_k81m3iHABb-w7cV#WyMoL+{@Qr3O zWNP%|V1$zuzfqmVJL7!BwvEXc64Bw&(cPU8q9i=)I}Dw7;XU^YsCi6-O93K}Y%FK2 zNcJG;dUB9lEGn!P!(iWUc8$1Iu>p3_MmQ%4xJmp1xr3WM70rusBNHv&LL|0S{5Q~v=C(FAgI>tRYJHy*D(X1_R|$Yqa=C{vxm=l0kEzK< z5}5^>0t=-1eRpi21zSvs3o~!r<(I*9PF)0?P2|nZ4Q8^8kr7!sr!^@j=RX%p28Rz< zNl{h8<#7Tc<)yaVC}26iu7no@{%dymhk}(E%eF}7}|&U5b&(YoVT`Lh-KO!4fd z<;?Wm<;cUi-V1b8zdos-pJo1-;Tr~=bFRlIlYJqNqueIO##LbLd4)kUH%8}|)l9K* z@%+8+jv8ijM?t!UDmjWY<~3Q)1F*Chk?*8`D@UzI_$%KH81e>40jCaNNNx)T*FaHC ziPzWiKZWYR9!`iASx2#6n_5>8+pl&&M*j)<(}uB-sVl-}7)56&>by|#zuQos2A!;G z0hZ5;+ft=6&k!ssDsx1>C!pkg$YXz_14X2g^0<=gpjRJo9atgq5$HGO^$i>9iF7R% z>+Vf|Kp3Q;(IoB%Qws*94}nvpj1c4@H6595%j!I&7(Dg=z%998?RHHg(Ue}|*Wh0Y zzui4G%>Ia{R8Pv$Mwn@W0O??cu2Z1rlTV7npJ|Rn+?d$d*cNKfv>T*RR`RCtRFwmN zN%Lsr6RRqOaNWv+Yx?kxvL659aOfa8s_@o`2${#74Jl(8Rh#P9r5$)wz&)6h+e!LL z=R+YMZF=Fy0Z;p)ct(EBC`89PI16@PSXbkM#TywWVX1BK_$M|<_wcQGK9V!7cI_Mi zm;J{i?|J!C|G&oN^4kL7E4IO4MSja~X{0NfHSZAxmf&p;6|Ob=?M2(=KXk14lYpoG z@8W7_!zoD$Lsxn@v#C0$;l;d>b*fU}@e6VR15w5FAg`!6b^U>inT|-#j9tTPpNEzI zvv?>dsJE*=Wd6r>hxYe}wGCeNRKi{#Uv!-KF1+3$skJgjqVV@Hz~M#0=MMx^hXHI< zp<-(|0CVS5-RousjiLnx@od{gbai!+C*+FOPJ{>?32Tn>*#ZTy*xl_OKnuA&!1PSY zN#m0Onu!LBLf~Cevf)$CJT6T*-GxJfRHUpzFAJZ@P0&?XT%eBn3@ytN;vSIj*rmGLMjZ+03A2b%%rq z(9@%(#S?&~I$rk+o)0T#9h|nQ;oSm|hGxAV6j%2GcBj~`;`a|FzH|c^b2{-IH2bxA zol?sO+|H?te@y>#c~KLldy{;Z1$A|CJFU9R;^r&acn}6_C$RzfD6ahVK^U-C*QBFc482@Dn>E>uw zl6ox0^UPv{!iP=-yM5qMq9}u}XtCNs534@gb5QkZ97wu8Ck2O3vs9 zLn)V^S}rT&Y0l0|fkyG+hs|2djYE=ZWN3tdEG+!YmW#j^*lTJDp-qkaFSr_c^0K#Z zU1@dGV=CKQUurJP!Demh{m)eL+!oK{n{Tn-2+@oD^muD;d$FQX7xP_JbT!{mQH2zT zvsAgCgQ@cqCSjZ2EVvX5G&D5jt);DkP`;h2jN^^t^p?|UMtbicepu|C*)N+=*^TZd zzr?1S@Np})8k{_q2YX|D<(>VLgFLbaWR3~iqiVa!mU-;@+VmCC!o{CjunpiL7`idp zFi27_&8Z!#iwQ<0)omYtB)jAz9BgRQuSS&F_~6+wk|;8v3ELCP4M(j_brF@=TLu&9 z27|`D%_NW3=MzJ@sAa2D2j7Y`+nql$OIq-}nVgT-6IA~V+L%__<5yzQk8;^PfyANv z&LI!SZw44$4vcC0BQ!6`RBQG=@OeZZK}vb8$CTe zG&VMt^=CR)keSAQ-`v}%n?8-dp7h3^W`FzLN&a{CQdxgyW6LZ){l$B4`j~1?wBcN( zWu~#$)=ZTR-@m^~&(cj5nQH8d^4{3p#8hjdjiI5Tt8<+W4Gpn- z_iozT+v9oCF0(Q<6m$2SdfxxOTEEoMja#)Q+BkpyJe{4L01OWgvv%!TIyyRHd)_Xa z$}-W$c$Isijbhsm_HpNJGx*HCcUI)*yYll1B}((UxRI(h(MHld_OX225u9@_R+$=O zcg{KIjol?g97BRh=Jjxs;f>wJE|<$OI5_CdEHE<$2M5XJa&bp-DV4b~c9%Z+e4f6( zKFU31Vw&Z0xvRG}aBjT3vAfvS*4EP2))sdR=Q6{GVw`g>W4y6D=Um2kV|UKEjEUu) zr;-MM|4i`P8-duS;F|!?TUlw!o2IPbc`I!qdG9u}G2d1nOJko5Ne6D_EwxGEc`GYT zdDD~?Ja45{GOrKCIOkl(cw=|Yxr|9;W8WNifN}D?m6fKvY03(ox6&$>H+5s|&N=69 zE{#X*&N=69!W+AD&Sgxh8oQ(pNxhOzNqQ#kJjEonNa~SvT+(Zjo(k({OL|q(>*4bq zac3FeROa7@zkl5$C^ucTi| z>X&q^XkE#AD)N)C?;Vk}Bq4dfA6^efB>hoRL(#UbkhfRT!*ORV+V)LJCaEzjm%X;B zY3!0}faic!!2Q7ElJ1W;)pdq}Zq&Wxy*% z%UwWM+_`R&_oeU;*bB@H>*fKyac3*am?zqKKL_jrMD$L$>7{BObJp8|HwY*Ei`FNy zzQA3;SztENT~v1g4=1g@Kr3);ta<`J2Ocl=xH$8U*wzVb3twN?AAnbYWpQT(wi6wc zoCX>zRom1Yv2$%lDd-F95=T-i@Bpw6_<)(c2@pOv#h2^`&jL>aSHkif;G&rghvg*I z_#R*Y_;naM`h}#KB(%O}b`JOf(UIAkz}vt;u zM{Fv4)-*MCl~|X!Yc(a!0{#Ge&CK43JI`n&Jqq-j*}>w~B_5iT8czq70GrJ00bq!P z47-#P#-_gjv;ob)Ork%zM@mY4$CBjN29mmohC{a%R+V^IYCxbltgNno|4)4*?uhGU)u8YC4a zU`bMq`+z<(+ZUF%6E*sz*7qUc4Ko`sv-f}(flrX!`WB2<&&2M3lGXu_0#8^N%OH7q zKS=bS{Nb?A<$*(z=2C&IX=?0$2j+*X3hIbXjO>g#(^ZlL{1P~BWoT*vnY9wt@-Jrg1keLK79N(Qyl(+#CEXWReIEE139T>D5X@3Z_2IQq$eYOe z78$LcJAp5SD>#+{Zx@Da;v}g9_!h9*%q|yg+6kXt?0Huhrl#P-u%J9%36+WIX8oO3Q? z(liv~oOA9Lcw=|Yxs36~?woTOU#hH_@w_{Xwxgf_o#~^o^pDgS}r!8KIZP$04H-#S4S65M>~rb z-q!9Ob}r5@___JH1vp=P^z?KU<>C4NzrgL{Zp-tETow}mcma4X`=7RN=D~`OueSAN z=gD!J3aiAq+FxUTvNA|ssD;ZsWk_;;MMZOI1>s#qTV>w79t8zS^$Q$493VJ-R*{ea zeJ^PII8LnVxhCrnGfl#oU$&pw?&{sDz^n1JtF~i_%3}6@YD*Pf53mp5-y1tleHzOc zT>s><7{2~z63RNLi=VPtI4ooM{+mch;_FY%wH+GP79+8H1!M<0ubtTePgB@I$9g0; zrq(${HBpKjB=0HWzXR$n+9Bd3kj5BXX4E1)z#2JO{y@{O0kYc?BvBPH9wi7I@I&WB z>UePsHG4E9JZf+TSTlCz=S&_S0)hN!PzA3PLZ_hVRD*i>Wbu5!1o)k>26C#kQ&JZ) zNV>M)5-26E#$MHy4^bhH4+16}1UyxKRU8?PS)sq<)K9CS{soQxbJDE;rIo~jhrk| z)F2)Sjb?7vZy^7`+&Hmx&*p0WaUh)B_vo^3NmsU8ZaW0%-szLw2b?abn^2Wi!ybRe ztqFrc(@UWp=(=bPx9y|Rcj8N52aQY*1mw0a?q7}qBR<$h4{ibZ*BYJ(NbK}pe4CK7 zYJf1`EQ(ju%|H0Mc!dsx8iKuQpoJy&-=M+e&?n^KTguc*s85MKdMNY=3`^1WvXHan z70O`Y(6RxNM$QOamv@fbNQ6_HaUjvY<9h;b*8|_f*>dSZJpwDQu(`Q6j8fJEhm0|L z(c$RW+86d_bnUCzP{~(7u27dSw$R&7iB2lu^x)j`K&NIV^94v_+N)YUbP6*&2lGG| zX5GGp0JuqGth2tz5C45ih(QTOTz=w#BHq0~D;h|uM8a?Qss?@9tJ@D*n$R05KN9%5 zN=66Hgnnyx8VQZNhqMWP4s%@}A-xpAxCR2?{Po{aPfn>NyZxM(IQ6r@a^i6i_`w+L z;YQkrv#z$rbI{fq-MP~q#Di&~2|I+Q_vN`CzlaF8i@A8PLaZtJ^}ORY3> z`6=`q?_l^z6JxdehJ|>tb@H)~g~`>QC*(&}TkfAu7R(M6w~8-t(83!J6L2i{ zAN2e{+CL><`0|p!yCL{$+0k5uquyDzz1)t>j2T` zgprYEPh7wdh$4SgUxRwJOx~A!#GUZ>r;o=04}eMDkk>WT*sqa~;0qyed9+8?NY2E> ztVME}N-jC)S_i`*IxXe3W;7Uf@9?U{H9@0Yho!EC;iKv3?`~x*M{HK|TmD#0SjgSD zzPfDQy*SUk2)Qp~Rewq+(m2{r8{C4%woYJYqNQXA5D41}>M4$l8kEkQ*cz}{UEvZ>yzN)(b#byzoz99q((0>F?#JK%JWusJi0tzOlv-4;}o432+? zPa-l-we=#?LH-F0i|wONVY<^ZJ^1_pX#??rq?m~S@$@1;44t9r&%_B@xq%-Rzb>C3lC|EriZDpmtRsq|)=gOO?nm177raqYk)f;oH1dXF9$_Kw zZn4D(cxNJL2#A^b$wQSO)4`bdHsb#1MAZmHd`(qFTvT*H{CMvZjOi-=9Y-8JP;Z(J z$+Ch8YAaI5ceFx+-R?I&R$swB8g8dh5A{D|&|AZyNNO#QXJfp%E)lblpeHT=FgK`y6)r8xSJf-? zvXt6L3xJJTfqbAOtZ`kK5R5(@<-Y7G1$#o<({x(`L)1y)>!tBCX_V#Mn~Z2lNIuAs z&|0dDeb3IWfcmS9>EonZfxkoQx=QF8svu6no3koCTPNR>lXZNlP+PD(Cke6F$4bhj z!5U>7r+x1c$_B2VzjFcP<>k%(^OQegAqA!O+Is<5ak&VrTSRFPTA;Q;9)CZzR=_jW z8d8$3ro@o)V>MsYVpuqNR$po@{=zynNc4{aV#SF0qz!A=lC}8PI(*3m5o)^>e$U9rBLQrm5Nc>6LZ7>~<9+ryKUdY(6zuh2mC#tAz|>XMT90ZBke>`5TveWX4{P-#K0tE?RtZ!aL3a@5$ZM)>KRqFCs6om}1kdPFNt&J|I zik_}0_{0a!9CG>}F+_OVsf4o?G|Rk&_4@0e=zeyy>2wc|etJ%V09u_|At?$OJuD2> zXuU`)2^v%7r2+?@vV(I@&g@NyIoKvfT4V3tPmGrzY5Wg;sMHUhvD2+(DCQXNt8g_JKR?LQn8%ki9BLH-VUM(nMeknv*qF z)-ffNwSNw5&d-SzU8e9V+HZ>wUvisE&rycAU5g)j_s)3znY*v%siy^&H!8FnjH!|( zpN>yt%=-<#@jbd=XX09xzoM+poT(K_OlCz6PbKh8Cj2+0)riM$3ug zsutIFEA0p(;g9+?{QcTZS=ci)mwg~jT46S;(bNPV%9rQK8GH2sujtwzO4z&ciwk1t z-P?wxCJ^0J!o)VF_LS_}XuguTnj z>)ri(v0OCuVvw*h+poFt_tgKOC|GG`W>4r?OovDmL$|Ftn2E}7z)5!7hU9kYiJh;x zbIGUCcIW&>&+^OGAk~n^0BfmDgaYI01jFX$54G59y9deE)qGvowcyFL)OaIhTLOcB z&snm^O%~xnYC7HPtFj0UH_IapL(@zFJ27FU*}M)&l?p`)@2_6w1G@6|+WlrC!hYTa714#;BlZ1vPmiyIR?4!k|m8*u5UC8TC#O7bqY|Rl1ct|kbUfNmXlUK;Q1RHr-fGW#tke;OfLgN-(iPJ0ZfaS ziI2RIM<{xfo(C{Q^N(zWnmcV*lCKXskTvcM$9q#*S!0Wn2Ls3Gbb2g{DG72tsa2ak zz7Jk{HP1}^Fn7l+;e+@fYnah)`GTKs%3%~uA3noSt78v%KwBQ%In#S!=GniCmDp;7@}Ds0u8aVE*n$Km0_WuK%3u#Yr~>C{a6m9Xs-)uKL3yQ zq$d-t^6q-%MQcyG-7+H`vkLp>MO7_ouev)P*+ZwmM3uTC-;-5)QZ1YVvjwn4y*_gdSqg*pZXTZ>3gAk70R64-!BWs{JL={rTJICHGq*>UVcw{HcQjfSjHPQK4-j z9F`ZLrZ6TijZ@2QmBd%_b+P%k41~j_~AxG-?g*3+}?nVd9BXzF?yLd8AowCsF zZD1mWOk~1`B1s9VajTI>!QV_OY!$)G)f+#-e002Mudk!?SgLAMgLsPYA^=_g&XHif zyFF!xkoB%ZD6VL!;V5P}@rXr=ySbSSH=x;7*3ehr>uFEekJG~^@)DAkVu=`(p2F(` zTkNS?68P=FQVH`A4b7R_%zdLITfWH;vWs6K*laIVxmpt;o>&Mpe0G76~ToXGz-g(3>S758}JyWGe#kXL)e7&g%63J zitIR7)&57bJSe5GEVTw|_t2}b4H!c>AFHeT8m&U%lFX8rGiMN-f8OW*eci&j*a9{Y zR^5v9@UcFFkk`sS*QGzq90MDD((3csQk(Syjzy1*UA+@Vu_&1Ud(^+cB1^lz4Jx@8 zaA*$BGBC@jhWgJ+=q0ffCnTXF4AN3DBGe%d)`CJG&x)!~er$vb7Bfx$ekD&dP;T;D z1)9ky|KdP-sC&zsgwCXRb{YZuo~q|wgIJ>SYWmyefdO#407k{S&iGw1ebgrR{HqZa zkDZgNr1SVZp6;y3aCi65_@{6E^c;WM?}?@sXl9!pl0zF-c0LKb$0uZaXxT$^5!1N* zC3>aD0JbkF*azR_<>eX*e^Xx$Ut$p~B~x?seZ6w&2rLR9c4hD|U88UxZ8HQjq)4qy zA9DZXE#NEkFPYxV_gPph5Go%R6IdUcD9A65`o()9&psD@X5QMSML%iJkHGkY~ zbEs~2qzR{2$j(4K)uzLt@c6_b{4O(mvakDtrJCYwDnR>4uxw*$?5DIJm~L zKd(*%=Mw=-zrl#6Vf+0sK>*5U(l?oZ7NA(EG?xzjDP#T^m?}Xxfk?VvEif)O3Y|i! zIcrt6FZ#vq$y%ajH_Qxx_5BGfWB8CRt6AKrL2K3}~Z(Ric& z6y0=D2Q8_Zl&E0DH}qojbaq{Fq394!x{S?>Yqx1|vs_SSxj4k1kRra=ug!N7-baan zN)wk6_r_AcRVey-CTWM|tA}gz)Z8SBl#jgd6N_pQ1b;a6dGj$P`>$bC%4^ z#gW1g$^Tamq<^?}tYUoS<1S~;T{Lc{R7i+HYxzmhyCOy0C8?}a{kWQ-3_pZ~19-U#joy;QS&o zhj{-__*~`Rk@=F2+F*6OF^it2`$JQa76pOkx6!{0YxN>LojpBSj@gsb5(hH0QrsS6 zslLObUYx8=oQ>W{l(JNqxOw@Ns4{UI3y~#rhHb@Fo>Jp2qPRZ!{HNFF!FP0TKP?eD zE&gaZoX7DgKhQ_v;zHV>4u9bZHPZ>mk!q8pH2uDJ5%Sn>A9Q?iQ?`;c$|bYtu{22} ze)izsZc?4-qH~09=GuZK9%N3uC>R6%JwGt<@OkT~CAP6Z;^5}gu!3Hwc+tXc>BUzH zg-OmvLv*cACdMwSnx+EUKuqB9itM8L!+7E-7+%l)O*!EC}n_I zU3_Ccx^)z3+eDmhTDRd66Phc+`(?j0`F$L=zpbwrFfOX^iA|}_Z|1vP-clSJ-YMM? zyT0<-2c7jbc{V-^y>l9n4@du= znP&M9n)+!cO{B0Qk-DWNRD0&jum_w0e5oh&S{t{EuM|c#78K@pSHOk>5O$N1Mg#Yh z)1s?&BvA}vDsqUgJas?H+~gjukaKOQ^Y=v*p^reogQR<#(wt2-fmFqto)+O>a&0&! zcYLUz%cW3xhPpzmgzHs+pxFJ06iVEkqV_zUC=GAvkp2rnH>Qn$=PKvLcUc!#y$xA{ z4IcLybW_sYB=#E3_!BY-s)Jiol)j@x zt#}A!xwXEaVED>E^O#lOv|gNA+qmQf!>o`_kx; z;f;giXZ(2qT7Qn!HoirT-1u(1U2X9OKwWpl@b%jto>N^qb;;QJ!0wUEZ?}Wr;+TQ zA2nYmavJX5FLZ=_S&Soms7m;2_KC}%7wEAT-Paa&=zcW%ty{)DdpxSr zf6Cm3_l!JAG6O2K>gt@0rD#_P@hLlbI@Z&rau}J~`Wjos1=;aL4h2DHD$dkzW7@{< zcs&j!bq*CbYtY=zzm-PMm8uw|cLpWux*rBa;}W`6+bfX%JJIZp*p1tY$@o4jv+I!V z6n4s>V%SEfEfTcPpZ)L%Mr9XHdiR8%2OLW@u5<|yz*?-0lk5J<+jw$rXccN3))+Jn zy@{SJNAj*Gu#D$C+=7cA<{|gl;6o*yJlgv!S8_IXcBPr=X{Z>1<=46kX@SlcKpiK5dR8hNsHd%1aaTyH-t-&w2E?B;upE#wn zAw=wEFgC>m!Ca=A!S4AD6?G>&i78G{dVSQG?fw@0Jl+#r-o7BRVjffLP^R zDSS3N!hCUmG zvr{)7%dtFO-H)r8vOR(H@E$SjKdi|jGr8qs6>$}KxWz8G^0oYDSp@i>S zcD%XmA#iq?q<(8G%8|n_?Sr=?dS&NZX`Gu&l(tIVqjZ_fOg*IG+j)}1wfUK@Afp(e zL0#z8T4ZU3JFBv`P{Y!VoGDT_$osA1+L6jU$Q3X>lT|=Tw}0N=?Cs^R^&h`qw`q_8 zjAi7|9F8kvce9g%yxm8;8ErGM?pY;Y6pHEJ@@6=>EtTr872=Ckz8m24x>1wA{3xFV zdDj!CoFW>6m1_$&M0pn0(DwB9m`h8h%?*kfH#zxEGWE{2C!+ZHUra_x;*9WJ?GP$} zxIFz_E0L;Y&rRXh3+LMn2iq^GNJqpXXIY#lZ1pcFu$j< z(Nr1b2xVI$K8W*L3rUqN$fPdyZCi^0s3odkQ?c9X6@Ql+-W^xIt+ldN5h+KptD~s= z)#%}7dN$#SxCY{enc>nb0;+wPM5NQ!(u9Sd;YP{pyTt4d`D2qH&`}Xv5S#|cT5p>+J zw+!0HpeXhlP$(`Ar5g*P_>Xj^MnB#a zA|#ebGg&B!vGj*W`E~+9qY_{6}fTAgd*d^1bS;?lB8Wni-{TYsb$=n0u_S5I|23Q$;LJFk9g+`5XdfFsoT%jF)x+^$@f*LBL~ha6JFLBXV{5Bf3L~J*Dz*daJE$a&K1agybaU zkC5o-yLu~)mfOh4fX9bEQiF-C@ec1N`8c-S4?F^2R;DieY?0RIzQ|(&2EMBFwWyOh zQ+1~aQ?-vh-M=f{PQ9q6Pp4YUPq(2D@qFg!f_9rfTJX4?N%S?>Z$MxM_+0?LSwNT* zuJi_GwL|4U?qD?%uLy;Hz-&KTK^?|QWY_Bd6| zSkG`-|GmUsH*di*1cq%MX>hY=P%(b;*do9rLNh>@idOUiOOQ7{kCQa7Lc%e+QyvDw z1PJu9$-Xqn3=W?p!!p7(P z&bqSOW zWkz_q9H`f^y6{GjT=(hG70%$g8Fg^*C!-Jh%gZb}GTLxV169nZ&fX~G`OLWXWr znzUHQ13$DiS`LeKZ4N9kvIhNe95)eQ`7zoCez}nW=(9`nsIJ?xu|fWh%B0s@Vi$KB7@4 z?RU_}hnudrlKod zqjkuB=gZur{bGu20X>itLK!E}cjvvs?fl?N!{*~SXsCN-Ih6d_=&6g8{FifTh8kV6 z+t4XZX1zHYiAdVEe>aXu&})8+XPLpxXL6hFNx?wN`=k{3EaUxwfbGlCi~TH%&R6oC z6uO;#RM>|xA6niteoYe7k8Kb8mdN@&BjKHjFN}NKX4rXb+=~15m`5;*@L!2$C4GSO zUlQS>`=)hV2WaZgs8h)*z@N_KPxN!eOV@ol2`jX!iGA+nZ3lPTu5dce zHRUl4y*#u6AoOHW2tBYvA{gR7G(H+Z-Qr9#y82{|kBcpEy}3*Fk)CdR;lf?SX^KO3 zJRL1dRAi25Tw*TbI;Ld9-nu3M$qR6wp8om4QF>7uOzEU%g*wSazW8kPsRqxq%!48^ z--feMAE%7j*Ax^43^#4yFYtJETMxx&3dU(zQ?oM3_UIUOYWT1CA||$k5VfNFO@;M# zfi+e^+XSTxygp}hAvx7#`|^2;hA{94s&F&4;q1&X9Y!LNezVdJ{WGi|ZCW|shbmZ9 zFijFvbTF<4&82x=Jl>7h9QLOgq`h*N=%tMuToE^u9`O;>`x~3 zo;wo#vQ11)?&+#?B%P^!KQ?NOinSm4>{!|iVZZdVrr7{iR&ckm7JL#C6S|ei{`vyh zA5i|_RU0*&k?mdw9#6{30OWr~-F{x2gf1~9I`#beO@Te3erQTMk^JtBjS8>iSN&+p zOx4dGAF}AM<|-qveT!z5te6GuWtCR2T5Dfs9h2<}>~?R(+l=MP;oHhBylZ0v?_|H{ z(5tz4VeU+oZ;~;963HSTqcKw_Pm}v#cb)RxLzisn=>+p(a?o)rf@p+VixGGc=Bhnn z{P(90Yw7nvR5a8Vi6-E2Iyipej`(qvx$QY;JZ`)nC$YQSlpT_9hSDojwb7%MdkGO5 z8;oPhkFQz#7kt*GsRePyL7!7}+0G7!jhChJ@QXa{dkLAiHx{obPey`@$Tmuv+M+3z zi207zo-9w<;-2fX;q8upkqm;0aayX}NMJiY%N#xgcC2f28DObMdnwov&JJ7coU)WI zpU2=eB%#d1{%5hhr>T&QnvwMz_(l`_#jBn@Rf?CwEJ=nYsr~KOPJ&8eiX%TBGRe}E zj<*>aQQ*WRgdy~&s@SC4AddWRM1)}U&W*Ca+d`vhTC)Xq3gJia#{B20>GG)7)4`V3 ziy@it{(9NqR5Yh09Ur~^)USKkM!M^M1iXy4?~CukmXsWF|4|7C-W(?k7(!_H6A?dN z6Pc(}zvO9MZ+)%ln1}8U<`T#;6z zFWe20I!)Ap>5 zKtKI{4X6gaFRoZk!3wLYGY3maRcDjVVHPh&2#c95iq&n-HhlU5pE`ST*6}tA#iDr$_$ar`RzD% zmNj89G^s=IccHNR1UuS~?&@rKhU|L1R>95ZW87U4QWG>gA*Wm*JZiU>q< z-Oc>{!%67_8KFTBSVy~tOKptsMERe{^^emk+^)xMw1?D_!&u>0LmIB9+9^n%?OOJPGPuxiHkh1ER<9b_sY=YCm z1gRfTWEnMX$$FcCxjQ2CR{Tdc*S=^JGox@)#=i_^A4iC;3UUfzF z+M$X*ib@l>ikx8U(`~=j$F7HP1qBj*k3zx+8m*aPHSBEk;CEKEzzN_fCvV)VGv(_0 z_d5CpAIm;ukv(C&cK$V#yZg;O0<8^=8f$cAkGT|?48Fn)*ifb7-B?WlU8PgaFPAYa z`*WP{^)Q*LSkB{nh}^+)7Tk(AqZ-)@oK?*R4|Pel#$I~v60|JLrKT%gc;rf`*|Jg0 zV;eybY>)Iu>NUgU@qa?HCukhbS*k2+{-}QZ;c_YiNJ#RTU<7m<>NrwYaawfC__>S% zu#DU(i(-8t1O4J^w_!4M9lp$Qsv)13Wg@91j&!|QF#WnkJt3nrkyEdC045-(PF)QGH}nWtPTP*gQBkUj(a&t#=w_UoVV#BOlzt$u^k?LOZW?k^r;D&Xerrh zx5)U}1^S5)myie|c$WSO399%p0qYEj^9OJH%icF|BVn+1RVsAi zTP6pjD=O`qo&-MmeJL^oZDxIzIaQAPc~RI56;l$l9&=i48HdL*j`b2B@9xb!#&Kw@ zK8^rv$`*PKE&^^Q z@>2|ezuzL;XjpTNrRl*~YaI$LF9k_$cqjDayS0Res2L2g2@^ou*^z#J1xDVOULl!X z_*Bb7V=nj6jkmwCP_k`=1su~@@#-yw>O;0~XQWF|RqoU8kmYjd9#r%_mR{n1=xAfJ$q3MAPZ|b?)Tb^n(er@K;(q5GW5s6 zOTC@^5H0ns;tm)A30YTCC`|p^aB_2iw(?Y9HaGF`@&#I>4Mj$#qI_Ef8l*UZfcBYk z%Dax>K?+^u?r@avta3}lRI6B_)D_Wjx62X{%G6^Xl#T8ksY_r^NRAu`aBqy6KMi=X z>9Wv;s-_uB#M(!$08IS1vhkSPRH)WK0$5Jf@6iS^67nESCoAv#`vQu zqNwv(9_{B9}uvM5Y5F-g>i zp@WE%BHl%I8<)fnFEvg}7@7Km4Dqty(_C4C&q05=$+;fX@+#yBUPhdhg!vh}r#A|a zxmJYvFBeN!`}AVN3_$to+<&9}F0(m8*JLPD!PI)ZP-fXpo(&<(n0S>yXE|JHMp4kM z75}g&cq2rI1aaAveR!aEAdvU_$@q@lcZ@wKLkqGB?h}5dJAX1DHdlRn`DH3U3)tl6 zw;W_+3t;7(H>kdxnb|lU*^qh)>v&Ac^jX_F*`xco#QU`ryPo^O*HT4whCH4Ra=!7` z>X#gYMyeEz6ME=vKP~ve1RSISPJ!fD#{MFSSC-xOAvyV~m^4c_dJ#^Npw(sGnKjOZ z@pCL43s?Y!)xEjDezhDZAb!W0>4wEcbYyewrg~#F^>(^u*iVFCl2J{9f>resJX>bX z_p`38%Vu1Voj7j|2V;Y&R36G9j_73?)bOBjk=!uE^E~$>Eqd)=a(}a1jMU3#(%I7P zO{L4dR7#=?HSf%1P$OBYq8#{`RGbl3QVZyj$tKR%w*kH-Oo_Cwdk za|v_DFW-p=8=0mH4A3YU)eYLgKNAl+PRERxYD_6nxjRvcq$GbP-Nw&}BtYCZ|&+3;Vsh7nnP{*JbnXMV9Y=zi0;pNOYHaVt z0#_>3OO2On(1nh5+I0duD`|`$wT+o{Uk@frtLI`{H})4mZLeEgz{rR%8$;!KHK+9R z3KpIc={mX}=9PfX!ZJr~@f5m7!$M?K&s*F@4jU?dIosfQ8JPfSg{^#y0(N0`WArY@ z@Z6ITRH+ErXl%8;J@wN~@GADNL7Pi2wU*81xu*$s#0CM3i!#0h+FUqUkpo3ISvx)& zXa`V$7U)3uH_6ahD;6Nr=B%3mQQ4$B3S+)<(WCxJp`x>aOsiblTOo66Cf!@+0MlFh2M?E zVHtG29h`#F$0U%3HDJ<_YG9l>gIE3WY6_6IQwXR zj+}6;PkU>?^gGM$;lNF-<_+emtuQ<`4)6o%bxq#kkG)Lu4c}9Dq<*+0oUR{ybx)5i&?lyP?W=PDR> zFoxFY{GR>7S17Mf;7K0;8@P$aja^8&{Bu%?9=>`0{hkjk^;zbDL$TO+D?~GiMZd2Y zVcE$DmY|>}Y&DJ8`m7vYi!}8%5Seo|*fMe${kX=bfdXqP^;$5a^|xlCYNWR+;1H_# z0oiS%HRq7CmHs;PYdYTkiCyDjHH_0hHbX+sD|lzka`BIjYprC6Y%zig$d4V;67pX^ z#qKg)Qer&Xk0AZqWmrcd73tT8@_#+OLebD*g`YHQ&|BBAjy2+C({F`7!0F_B2Q#($ zP@E7VPURYZHCY5k4*K!eK^{=8{I~a{8-bD{B|98VKkD6umJkiO`k((wM_8lQTFB}Y z1A*r&F(bBSBe;^*dP1SMU!bf|1U`sHn8^0%fp3W7hyY`^Fr`*za?2%rt!e4ISoKu=99O`5&T z!X>P}NJI5T$^ui+gTkm~o6+es#?G7=B0VO+QNz=YSP3%)8e=BbCGb5r({#pZtM3tC+$jP&qJXUFs@!y zSfu+c8{GdD7XCFlqAd>JQxDboeO7z~kd(C&K+-`=KxCJT===9}d7PGd9x)uK})55+OgZ{>2-y^pTLNBhTe3mX%rFeuZ>X2B$8wlL&Sk` zxwpkGT&3XRV6?yw$*jZfT1a^_imFdiy11*F$%&YWrTW`-DZTIz1gKjye zJZvkto&);5!cXqgO4`Z{KRHGmI~zh*I?>FYRT;?Y+zj_eCUYGjQcRuS^N;Y-X(Y|p zm(DLk@lhQjI+|7}mTt|;6^KAn@)il<{L_}8xl*i~8EO3pv`S6b>W#YWw(7dU3mRWE zc=tke&mpU>(f6GvUNe(>CfUNGgt-=Bp>1yg6p!$z(BfWAKZCQb?|||>Q6}qE5|mS- zl$k`tsN+321>>{!g~)m@e;t`%jf{jMO#-;P%nu+aei1&5H$-h|&j{Q9a2V%-KZ|81 zFh159?dnXx4c7P95k`9=i{An1R|~9PIGs6U2UZPv2Ylx@sukt+K^rr2{zhl>Z}?okSJzxj`-_1goSlpNn5^P1S!bog+^INz17=pl!?!D?6@K}uFiBCTbwTb zS#FNf+nw(R8O=u;#(18m@J`Kuf6@h&WN&{H3B46jpsESVBk014&79jCmD4n2H^1^UMAGbeK4Xy7~3?aHG{qzOW7>aURcGlWe&*N>w zK1jbE*Y0-!UudBGn5uIzc1li9Xp{#PU~=<3kzIfpMfWCIQS_?~^18Nq&d6J@F(q(Y z=U#hD*zXZN z6C<+%N}=@b@igKKN&42BP-)ey6@R{?wHc~SjtWjXO##A11m;rlV*AeJ^gQh8OQ2dLGw!Jn1B4WdNn zHJArzn|n(>&K%7@#(V`&PbxCYb006k55*5F8g{QCZH<^Qy!%f!`R?nsj_5`B#?XZe zH7~zJ(lT|ak$DUs1(FUJ_oIlkQA(mzg~YA^dWY9u45gDTP}5h}BL^QohjiA&O3Oh6XI#l3(I3p&*H03?>dMK6)pnAT*qI>ZNbLT?PySQx z={ea>k|an)l@uLyXlJYcO`qJzS~Wt-sN#J|nkvH|xG6~};va*B4$5p{6dwu2ziL_j zsjQKH)ULYj)~&Y)gK+lgf9}OF!V^Ez4ec5xllOIg>*r_k?%mgkmZ3Z1spX8w*;3sv zT_}SIBa45&?qx>(9l-hjqRn8w&b81l!GgC`nLGa(b-z_*@`>Ao-z0zyut@m&c(ih=C*JF1NBry97y4u>#lT}>_y`ACn#)a_%P#ZcADO)J+mJ+DH_Dy0@dN865Bxi_q7^f z2Seu$D2WXf@G!38|e*HsH`(nf5l0lP#k6}jmLPy&d1KwV;{F#LqWetepX(=&Z zz6AesiK*{U%Y6O#m$|f6hUKbkdFXwCexDc0=Dn>K)t9?D%9!5XCcVN}Rb>^JMzsR< zYo{1wm5F;hlbe2h0-`n8=X<~E7W1*kX6KwlhssG(;=~Yv;D8@Hq~CMC6VGg zBcq(6{g5!GLQ&Co2Z}iqURa#HwMwG-;9fI4I`oQ_gRmYY`=~Yku$7iUUW@OQ^f{}m zsj-QQ^(15eo5$1MPXv2w3ud3!Su_+KxFOJ+5r+SkM`)rxFtHNcpy|6Ro4z{0=v?Xi z-yH}G)EGy^V9dEigY7kYMdX43YDitn$v&v&w7-`$Ehz#@*(71xkyzI406*M&97AVibiOkYyf7Q+6s-RRTEFhf}p80+f<3S=4dQX96~DJ&hoU#K0S zv1Z>zb4M1v%>DkoRb==(Sy}KGlpGd*M=c69r~;XiYn(eqDn0l2MeEO-tHXOQDr+Ds z2+;;BqK5nMaMP1Mp@z%XXF{tYH=wZs>OVBIKF?R^wA%Qeb8R z3j9H!qb41_pr|{c*&FZ*CX6-oIP#fskQu$Swzl?{Nc>kpWNcpzxj^g}&-J;s*!}+v zRZ=`BpDU9suna4Lqwhlz%O7Ght#(vJMfFSA`RsHjcmQPv7up znUX$vDdak9IJ>nzNy~|2^6#=pc|@fAUIr-ko`FN4R>|6pw#vSq;;?S^jSd?z16uB{ zffbI?QRudXTsoJpqjR;WmoFgn_v*#iH!B+(7H%E#TFi$3r;RfYOG4QKIG|~pnfImM zQnOrcE}fmox;IT#)WVNkx5P?xKi! zwZ+{;!R%r8{qw%>{X1vooS8Xiely>k`Q|}aLEla5uArKL;V0zv#uf%<9~L?bqlEXcsl5|UoP0jhqezZ5 zH0xu6z&A9a^Q}7DC!7Z@Qj3jLcfa5$aEJM;q1XYrS9N@14v+KW->Q3qitL))Iu#a{q);y-Js z#ym+=oens))5n~_n)+~}xH%V|CG*MkUvx!J;%fWz(XriUXv878FOOHH>>|0kNAcPKFdM2XLc@x7tT}P~gch(fgkAnbg(8r=x-r12E=)lf z@}13V$Sh3>EtfeSY^JhwWKmdw#bwV=jDI}5&F{T%yC6(fS}`cWnbSg5%ug zri_djd7w}znWoh>e!}haw^L3@Z3D8$IX+NG=Recn{3j)q+8Y~30XYx%Uius*i!p{~ zy{CSs)B<>7EvgrUnSQsct7m$M7B@4Z~$Z2fYo(epg=@NuL}2^5&DBT88r5+h(zy}5h}#ixF+mR3608`jyI z*tTY;WZ&aG(d7Ry9EDP^Li-Nte^Y!-?=BIri-0Xy+lQGYek{eO#l?XR6ZZ>d{oF>E z0x1b=$Tsg>`n*z0!z1PGcBSZ6Bc0d*Z_|(N>{#J0*hQY}MjjQTD{^5h11UZ=IoW5q z;x&49(9QI9h-mLQ)2}md z6za`85v1Jbe1&(9WD;nhq;)-jTLstR@_3|<31pX=%BWHNypqPOm9L!tu!d=s(H~@T zmk5W@?}I4FRj?!-yEA2zNlt^qM*n&~%|8$@<^970lL7E*>jbmDAaH? z-5kP*c4=_nN0ausG!%EQU2u_BlxdDEyt#F#Z{FWJJs}0<$ zQ90VFhE+`SdZkA4@cmF#2g=+-roE-q(bcYuw`QqQ>UvFnS?D0vqY6XI7?ok-oiJ^M zO5k`Lp(p9B8kU|)OiK(lKu?&ib2~CtECvEP8uO~}u2^`_)RK1TIFJOlW z3sfR+1B+TriWE*l7}eUqGkT%9%)PSP7w>TgKAb&B$kz#YdhnT#Y(<1PJDvyC0qp>V zz^$V?g*mH881#j}RyhVVk-b_^ZCkH4?0V~(HIr4G%WOsN^+#US{>u+Fd z{f}ozhz}=OB$grCo`2;97R$YIzls_d-a;*hv`aTu#jmgN?OAK^K$>#+R!azwelrcj1nAu{HV@HsQCG&^gk)eAU0Pl8u--QLDuGkg7VP<}Q3v*971U!CLhv2rcD#`f%23Dyb7 zOXuc_$^_XYFG8&F#K8yQj+^QyEHPj2OugB&o^N0cs>orMo`&uDy6ntpp}Eygj``sl z@lst;jAIi(^XV<6(&LyY=pwpXmNnU)n#;!#>LcPZb5bqvD)1qwStau4)Vm5HFHMFu zuRXVnDB&+#`PauR&YWz^1)L+ac+kKtM@#RW=R$S%0k1t>*IvCYlwS;7)|x=CJeUcv z+}e6{HC_;u3r!gOc+t_?409zGslIJwMj#t8$%>F9zon<+T-FWIhAk*6=!m$}{n*&9 zpmc9SIibp#184tCrYx5qOBi768QK@3U}fMT=H+y*vvOp=c06L{i+ z8Zq?J6u`ry3$VlKbEX(Wwepou@o;@u=z;s6Mpc%YnGH!2aDrqwOYPXqBov>o*X$DM z`t^1y&a1o+ZTt?Ct_)Jt74h3xI~S{|Zhd7qIXnHnr5Bdpa0S%0)jG2R zXy+1`9BwTJB|WP2Y20UNmeBCpl@tcp%6dNN`-+J+jbKoxNwrsenilQgH{^-LW4Zxn z9Yfr>V-CVktGI&cA(K3!XB1S}eBkQlruOT`Pu7g;=6j-7(nR4>OuT99jJ-0jhgO6s z@Af(-ZNx9oq`4P0-z(x5K%M2?S35S;{I4CFN>Uj3V9jC&Jb%4~a3h(=s= yqrMEsGQ~7dDDhTBL#E9<0R+1&Z_1V8x0AcbDSsP}~a?*S0_*q_}HvhZYDf!6`1q z?K}P5x%2&V=Vm68$;rvd*=w)8o@cG~>_mT1kt29Y^%M;ajX*&jqK<}!UIsj$Ji!5u ztY$870WVl?(h8bSfQSDRizwhdp0m8J8yea(;(t%{iKW0h;GnR(jE=j8la;%dsjDTL zgQ>goM<@4>wq^{TmacBLPL2$G+^rp0{hufPo6}*uZ#Is6jSf`(xbu7q0HgFsrFDy|EFRH zH+1o`ZB1Rq;6F={#mXfu2A1@6riInmQKD!F8eXP?6<^9oK!?%Ud#f#kyk8R; z56dcsEq|Ez}mhTmk62c$sZi3+~aD3EEn9z0U7GX?bzfzA8 zHIN_NAAB3)DMgNYh2nfX6A#fvXrXMNqi6}3(`fzlBB{OA`8GFfm9@_`J)Rv?@}fyW z+DVf-Vh4z%uzwjs_+&PG9+Br{pr#5|B^K}r_>>1U#{#wwF`lAZkz7u#J2Yki4?_wy zA!#uKCLkZuGs~;0NA`!u>qJ^c@T5$k4umXfAiA#a4=Y&Xe%&9v_UDnn%{I0EUVFFG zkqO8pX22e-zL9TTA609=6F`!b5~2}eT)JxpJulf+LYbkw9(c}fmd72ze4|dc>EKff zyt@rx*NUh+lw@E`RP9g8{uXexA>`twU+f#>0;ab^CPEx>qcxe^PU3r4@jZ#wf;3jDAMii-ozHU>@_K(0wuF z%-aj?v3A%^;EyL?5fw!IGff#hO)F@Wh)k&@mE2~5Zn($e84=1^ylPN`JiWeofbiKY&U6`bdT#AHD5vd=%S=2yqedmdqK0o@OgUBlz+i zZ#oMO;J0`Y3Sf17e zd&Gz382Cx8Zae7Ghw&sP+Qps*Y9V>L{-wh|Zz zw2s&tmqPO_4Ax!%D}wFzJy}r;9Dn&-_Nr|#Yk}%nuu6ztj-6vqeh z5ky>by^bN}P~?r%KypI4J|Oq*jKc(`F36q+W0~lWrc)i`Z@s50=>D&NhE2kxHC-Um zr(F;m(Nx!=^xB^B;NI;}4S6cH1w|qee5mxmPLY&*vvc-i;p4QZkEBVsGR%xupE+umwW z>k}J&k{Ea-*sd)Q8Xyg=;oUO`X$?xF=mVCd<`nlz2c9#ggMC1$Wr^MB{D5HG6XdT= z-gQR|->I|3PA*JA^a{GamRT^^-~La{D~VG~;r8&(~-{kf9|l9T&GfsFRoO%vYr|jIU)n@Vl<*y`G}~VXn7*gb>`z|uVBe2c{Y$O=AvuHY98hT82s)o z(?wjS5b*O6>mh9BS_&zipu>kj?s{QMqR!A+!2%! z%oaV6^bnRrtfW)fS?dD5Cq4_mZ;zr`ZE(S|8r_kEHN*1G>^^o3JKa{H&l=JH$#@nG zW(dc8*Ay=cy~OZBQ;ivT3sL|jVS$7VA#{J~J;|LvMTxB$8~5a2{pNWgsG}#O1?OB{ z6Uqp`FGA3v#4jR)^LoR6lbMiRJ?gJ8z;3vYz2bDfV5IQco0&{EKWq_Q-}mM>I+Hn5 zSCXvTF7-T&UwfOpt)y-w@!oXiLGU#5^Da*=)a4CbcH|K&l{mA@z62%xklFatjepC5 zqBi0CuqAoqgSw2Y!ESt#((@rqfx(BGNu=}iDeGMm_E|$F<;HdY(AQgd0{RD}T(G2R z{hO)Q#~JentCHRKCRo4lbQr)z2%-$liv{WvJkXozAzsKa;CrBn2f;mIKJG#=^=fBFK$`gC6)NrggKT$qXG!t1MXA&_1Z^DrZmi zJkbZNvs=_k!`+0{>>hG`#>hV@Ch?Kt1$p<%1J1h$MClLNX%)rUXx;Ir1&!4rU1`9* zcDoL9Rr(6F{+$T?it#*TJ1_L%e)U^y0i4r}RWB=Y_C=Y+x-zM@rpzpHY_B>GzZHT+ z6+s|`7!UN^IV&wa-mj^&PX}=;mazi&KIyAPruf(GsEsvgt^Iaa(yHePg}_rfA1v6x zqnrJ|Gln~?>KeKN8@bX-EDjB>_W2u6WY_kcMU)I39A zeojK4F`PNB`jJnFiGv6wn%BJ0Qc{Y6MocpQH_uzpOB~K=OnB5mG-DiOqAn8f3_pPph!@(7Uj#q(!Jl}41JU9PLk`MWRMDN9_V6v;j?9vRCMkt@pNpw<(m zT&35PoT4>WX%E=x&DW=@k_4EzSzrE|P>NHzx}xHRq@DDcb?TlTcF=P-qOqz9_WJR( zO|`0;%0l|02A+a+xxfumCwJe$d@DivufV;l^{-$(?;%FdlW4)!Z{Z~3r%z@iyo8F1 z$}3d>U-)El@i9Y)sJy9hc*Tp}|K=5^0mZbf z3^ScK@mty*B7>LUdwcrtFd^731C$m^jPOnfE(@|W!*0cj!@K2V%U7x{%So_L3kzTe ze3@Zf%zQeJySe1VJdU0yHTC$U|7#5gaY@!l-`QB`#lfgKxsgp^;E7D5H9Zbo3t0!# z!*0dS@yJ-`!RpGJBZit4!6t)`g`GeQMJ6&%ZCA`OUdafCK?wkrA*=&N*XZVkLiP5y>SmLL1c5s+~X``lsx(` zf+v0#a!Oi;KOFR9d@t5#&!4??^ON^th0KFvS?YZb1lB{Nn$0TD4UxuZtv_JP#S|n@)HZdoKSdcB zx(T38ys5;;%1!(EEr$6n$Wp_8i2$Z~BM?pcE;7rLUfFVACx!HFfbV2~%!^pE%zR87 znyagw-zd*+ua$7Oi&3d%ZFrWF^%0V{%Q%{d`h5jgxZ?wD+ zNTKJ+gd8qCGdLhv5E180rSUG7r0m10=z}+Gea*$6WP$pCGcD?Dgx@N6_tP_Jek0ik z+gB6Gy>=&S@7@KQ5PwLJl+fJM0`7(m<%@DNp*&82af6clAR8lkkw#Bii$-gY>7iyU zU?(%p=ESIvWJw%d0@}Wgg3snJSsj{xvRGAxJxiRCvqYMv@p~TdLD^W9Y#N(=X_buI zVsRz(J#BEqtISR3_;hMmcV06k%-D;KhQ13Hh@*3um}zi$mW>CyDTQ5VA?aW@UXSKp zCA)HU2~CPnqPb8~?r-nmn&;9`hYA=TY+eDHPc&noJBWAp!TCj|B3TA850I}gH;Nk_ zKDtwWr_X%fA<8;=lrV_jZ2fdwTs-tTn{ZzA=5*MKpcoWQy282AS;RB8m04;}dB3CC zZlKrrI9O#qI-a&;_*37S_~q^?9Ko!MHj;((7nDKgdIK%~W5LW;`HCa|ZT`5yQqd2& zdFI$6nGc5-m2EvT>H9B;*N9q|m-Gw<>ng@AJB!U!_EUxj@8^>a-75nsVtLX@UB@&e zozF|?7M8w zPPx=UOV2esWJu5Tb-qk2;KZa4-zEq_Ar$|odqq;WZ&G9Cc(YnM@31mGuS~1+prOWl z*eLJLN?-*(8&Y=jD^Jq}qdinh7RO_3ZF4vxDKi=2H!)vYc0n^={{7lD}VbWODVU;_v44c5ll<=rJsvkE+K3A| zXxh9F=j^yO-eCRV_K+r)Cix%^V8b@i!AJqOuDgAi4Z~6LAhj{BJO^<2l!QGHuswo8{*`HPXmjH&2${R;kjx zYpL(deBN<%Etz;Ynq4DwvKx24C{Zt&p0NMke{ps1<{s|Ftuc0_Fht9LeK>WtGr<^X zOp`L`B7l;S!Q^@dZQ@kxp{pwbJ|6;*SI&rq$FD~X)7JVr2r@6f=M=k zBcg4q-pelGj;;q|8%`$vo*M)j2t5eaAAbJA zB$0r}^fZI_8$(IcWPSjZzmban9i7Vla+^ZM_lP?sHfuU$lGoP9wFnL0q}1e(SoWtD z-hY389szk8aBDc+&j3)#q9&(li76h0`Ekr2sBYJwG@+q|_TzGcLGV{@fK=;;5Iv$x1nfmDg&+8=dexNV)au+be_HZOKSag94{O9PRvHxk5pUHen5{cQfi# zeAC)igTxB}ceDlObicCUTh^_lVc&f|_cl4^!r8`E=i9r|?N`x+c`X(%=Srg&FFL{V zg{-F+PIA9WmEYUMAN#IW^((pcnkNz*%mzNV+d19Tjia_M$*XJ3g?1*2HlvVhy~R1| z6-G(*%)3#41j@eP*XZaQY9;(^akP!!f9Y~E8bj+}<&LYNeONGGX7CpN>@uF(Ssn~A zYEDV#q3=6esn|?pyJVG|r1T912;+2lI6xKun%^q8_M9C(!+Cvia^zz(^puWmfZ`ShQ1scK>Qi@SCr~!Vq(85NHFJLJS>zI_?M(@^5 zARGD_YFClCx00bKlbSA6LDhL+emhwaAiEJhwEo%%i3o8X6CUNWjD%Z^2{)7Q@NgB| zT|Hsa`mLngEt74a#`c;^3z$QL1PD;*(CS{O5vuak77miUm6vrNkFFKI^*M*u1nrc! zC^Z#Wt!0t;eK{TQ{esp&`b z^J~4C->dA)TKJ4P9>Jy~QDQ(-jHzLbaFMZ5;pp7>a+~wsPhS7Ganv(#4KMrbAs!r0 z)JYiK7J3kUNw5CJX@A3(OAl>IW{Znz&AcWLz$1(daz%ywKjITl+5@+J(nuTq;kT>$ zj90q0ORu8ath#~1cE zztJ67*K}z#TZJj{F#E#B!79*8ok%6WBlA?yK_OQxGum$$RpHlmm9?%(q+M;lNpA*l#D=PtW-1_Yuo2#{-rq`|-N*AZY zs~P5CGwGE(e3iqMyozT0Z#OlAV#g=oz-6JtKMfB#`R9Y1u|?r;G-?)Z)P8n|R$@oF zjI#?f=1qh#=Ws#bbhoZ7D;3MVp&eB6206#+UgXCIL%BIqY>9xckhdH6S31t!1uUYC z<-bk&ov32|VVhGug+@ugi6j_6mb25%Zh5aPi~F`zWb12tre0EL&O`_8yXM+C?ZIa5 zEyu-ob^ZOvHhM(S|HTX~w(^-{`cNt`_w=63D8#;nX|O)tK{F=Cs6%I-X&XDhB3FWf zAj;QE3gX^Y6R&fcP>QlgdV_5Z`LYrGA;0ezPI@P#(>`*UO7)KnP&RsS?9Z8}a1g%x z7z;HMOHE*&*OYyv8Xu{X5jz>R8>5|wK1=wcl-l3lAz|FvQ)(|kFvrZ2tAL}vJcH_EOXn4`0_j@OidWEgDOLD%|YkEq4hI|5Oh ziv3Qmt?6>p(b1jxe4MRAGH``_qQ}<0@o{qVA7&h|a8KmoDK(i+nTmf0u9r)ep~9Ck2eOxLS|9MU3*Q^7d4B6&C(7FXy!-`! zmglg=?Yvj>AysTh`>=a@oUU<8qg8^|hd*FFwjxWyMsvD(vB#KOa&gmm-Vy7)Kb(4Z zJtcuAcDbb2uA?{LVEuO!M$*g#&f#w)A<4_{CFL34YEq1Fm9AL)+(Dj(Ij65X>QOPy zvXqI?*G?>Ot1n&9KI6BWZKDRVVplh1kMoz$*@Fd3qn%fMod5hdo>aYU&G_fN2)YKX6QJc>i$|9phB-^)HQJ0?bn z6?MHW^ti|smGMY+!Yu64xRfM_8H^NtpV?BC(UJE%u}s!udkthLulT;etECpDp2WeS zY@1Ca`G%|QlBPBh(sTDcff92&*9-BjJj(f~5uKT(=e5hcUf!)E*IcQ>V$elWs~hc$2#t{-)DZ;eUL4Xwk?hzbPa<`%+!^fJI0=Iw;rgh2)-tctzhgyXj|-4K~z3 z2&AU9o%UEg>rB%W=kGC zwj7P9>XRH-_O$Bwq~yGXee16AwH~vgap{^16~*8!WDspc9~xf$8i|T#8$s$dIBv0f z&GM2YD=36GESQW^UESDU-L8z%WvG2eMNp4KD1&nFP8Nj@M2q!dq^ zVYlG@Dadl89TvH`Q0%aA7N*op&I@je>C8h14Ihs!9B!XgxrC!RvV3;1v-`FnKKB4v z;CG0sDKbHw%xP}eMSQ_^x*$Q>mq?$wWw@;I&l0wOoj+~|#cbY6!c&KuuFq7JKW&HuTM@x3cY5!Cy%3YSTuA}JrBpw-B zii$}7q?NyPb(!t@n^XV`%;~RU7_l9^t)!XwJ}n2)JGTA~yBiaE_HZTl_N1q~f_#78 zlFZ}rj9#aKFarOzcJ_}vs=t5zhRXhN*|`0rZiRr;i=Vkour{%q-uJ~n=Vk27#2gb7 zix%D=-w@s{7BTnz3j+yN_f2BNUo&2I?`bS&wIxOVfESKzu2yvH^gAVRMU{`X3w%lK zlp*hwqwmBdLU1)e=D&4oXkyWdWYpgD)8me8yIQ zE!Dr#722gdT|tTRd;#a&;s?J=^_iT* z2Hs!ow9@QkdfOupZ!MxP8qb6D8XKzB-re|1aB){WJuF-^gMyHvM<1th;;~=Oyu2bL z+9I1I0=J^G~S*d|RI*3{@BcY6^{YlV0$4K5%qsON!@5T*covN-d zP{;lBUz+A~EjJ{M*+eo8&7?qf65vG! zA)0pCCzIFPE-%F0e;;FrNo%_!woC!K4%!$ zOjra7O$`nC@^55?a`~|-=ajzIT-Q@Ls^Chk8M}C$?ba8UVlc4KZyg!S+9Cz#V z$3q;&WqUH@;o{iBj1md`WAuwb`IDkeMjCyswPM5AHfr*&Q5CCoq@lS293Kq&e}JE*G1%>(z%!cC-XDUIuoy<#Rfm zw{YGs?TMQtoDRoJGs`H`|woiG~bT=cfhv%tO}H?=V27Rr9TEQdN2l zW6V;_tXbG|%Wd7(Q^%{al}3ScoLTg4_y#WFgJ68O>m0Q|61qhQP%EAX{7C z+Z1i|hMVLO+-W}#Bw@TPE0R@4feAR=1 zt6dUUUrL@hH5qr9<*>=RY>HA%Lmt5a_;Vg%W+uT3G0Esd35?y1gGUIX&NQ@%m;G;$ z%l3Bmr`j0<^Q@Qk?%XXS7Tn98)Jj=c2!(*Nos86xGJYZFo1_Cwsx8Tt@~=*TBA4-^ z$q+djW6lEvV3HVqT#tfr3H0Fw_Kj&_>4$t(Sa2w~gtaY2pV#+t*Bf8_u`054S^lv{GuIJ3wyGeYoH> z8o;ZK3b>U{q+|V(n(N@p(A%HgRV5&lKKw-YK;M)RZqW1?2!$@;#88;-PT`Hc?Jo4n zcgDb{nu5oZbgMX)gJ27e{N#a>kHRM|U017y-aBWc0JVKT-r7CWzNofqHaJb}DDsmi zn(%CS(N|Vd!k{u$;jG?yeaH7fZpqR6Q26bc(#{0SR0Z6F_z!+!@uGAR!PV-LYUwB6 zz?`LxDk?<<%T%((yJ(5~gWVe<*A{4^sEUBFuE)*aE&fB3G%gMCbIlZi*_MXgpoi~z zT4W#%n3&O$+M2poo!5%F+w{@&a?rizc;KnuPxvXMr@XtWu2EV};y9J?_LjW4X37JV zk}X&&aaXxpqQ=@ns443Hp>3ZJNpAGPuvxRNS;W0hQw!#};<&X`)ooD>)y&xcrSr8~ zy{Zye2WMZMq8X=gE2dPSt-W`=&4Z```uE06|)lFRwXZi6n8YxRnwj`~`pH3Wa&oSkmgfgo9@G%zk;4_l7Vr6Ed z=Hb$E7E!Jq=n6mH`Hohrthg4(bmsH;WAsalc%*p1?ocjp>|4++Ccoe~)l`T_%aCqu zQ3CHl45=0U#%IQ@BS*ZblBbw40~DwgDY4@m*aOp|Ti*|=Di}K#lR9#`e*d)@vPxOp z?EGerb$z#w`uq35j3_+gyVG=8sr*2yW&g=A0d;j$<2q{R`&U)^l!By8+x8bQm{QhR z8ds&y*~7vva!X5;Gd0Z$0t@itSFwXpcCWu?Trg}=5=ansQ;ML<*j9ZTGN~#YZhb?9 zYKVQ-ba>u$w>67F_HllAjE`I*{L}smZqf3E_DW@fayy>r6Bhz9Ou~qN5h(XxS$B_V z5R|Bzs?9^!6D+*a6wrEc?DcvA`j`_7=MB7f{mBri3ZB~{g zl|!L}9@v_8DG8`{5+41zxi+(91j@tREtd2j47(0aT>A@ITE(qN`p-vJcM>u!MluI= z6+4QS24a;ij27xyO)a~y*8h_=_lVJv>iH5avW~C8H?(bj+)K`1V2)EUHj9wyZkf^* zw)maawqIIOUUO(^^&W%mVsLXETfgZbF0khrE7)bhp?GmF-B|j+C9q9tcFC&-W^g}- zSLv?p!)*gb$u8!3E=i25;-o}QUx|yP`LM?L(#w)+KT8Ff}?iL zOvAapdZI9Xdt0ujU3T>(<74%~!YHzE!J&FgT|KS=S(f>#oAjdN8y8R!&ua4mcYl1wv3_p>_4Zb8I?N`09)cSpX;uXD1Q+f>VV^L(H$W+sFaFarYbvrKE;Gh7*F1m!c#ns0 zhPFKvte4}IcmPMuI$E%U7HIDs+^74p)yO&n9J~Vr`M5bQ<@r+T;TN-Dbz`E4A z{XNN~azE7p<-HC+b8dUjw3FNXOE8eG0R{F-TQ8P%k+(Z7X53909Vqhb?e)c$emH-w zXV$Aj-)UXhO3vTW{pNyQD=v|E+B2b6)-~(`iQ*2flz~c4XF^bc-(li@CgXHsGA(nB zj?}gXX9Iha1S*nCnlaHPCm~c+#AQ-x7LW}w6t`3eHjHX|w^xk>2WvMbgyvfzOazqi zq_4(qw=nw8R_uTVUDNc1hvSp>$G{p>pi%qtgFyI(1&JAr=rsK2Xcl$~YtkEA;_vO( zF={5$QwieJ@Hzr2NJV>q&t7$>!SR|wpZGQAOx|oLN5YXTP?Q-bbK1^RP5Yepd|cGp zeT(8qE8}(IiR5UZ|GYb2y%P>1H7WdTq{_6Y=$$p zAx7f<+1u-@hr){kY;oV?oy73Fm6&-&PH;7G@tt@1&_z z85H@z6VDn+->N__qvXAXjED`*>L^R@wU4kq5?mIJ;k&u4#uw-~aFttV9;mdWuw`?! z04?~Hdu}t&I+3;Y_~S$x+v&&YpZs{>l_00U<-ptacYYW(uJhGQuj38JdupbVq*`Z& ze=plvX{IJYtH_(&g0{^4x5q4KBtK1j$Eo>xsfPb{d+L|fp>7uW4-(mgjl!?pS0^=_ zi@a`_y7nZ(Jff*8NWbfj3T?N;&nh{hg&J+|qMjV^=YHgfP9TH)$BVD?6K0PN+2BHP z7I;1&pokP$Gp-cWD{P^}=EfdNr%IyPS{L zXcezJ{`OS9rhAFIIT9C?;xOBG+dl97f;@Zo`EYu7Uwn_#g6yqfi!<3lQDY-bBtOX7 z?rA3WK~k$&&BL$nk+w#2%bKF{xHwaiWkumh$vddS8XV_`*M{C3v8iIWOM0ymmL@-x z4rzr7I@7qWGga!3;@?-fMh)cXU9#PL*FvKWnMe8yVRjRVbO;4U-38;3E*HU4x-B^& zNjjx4IY>FKM1SiQxu%CgMbhdM2JVg;Cwl(YWeCj{Bi4=Wk4V2;kHUs!!FRXL@s zm4=5)SDW}V{?RJh@WyKZ1(c$yzfbj&D?S5ykcLv{)Mxy;BYG_@l$`ifiLmLJw>+O& zC~<#1dkZ4LzEy$Xl-RlKD2cgumHY>l#FlUv6V@Q6X{3p^k+J9s7Vv|Mu=Qu+e?8wD z-I}Kp2b(4Knw93ay0eiJ5Q%NJ0^N-+VsfMWX+tdz5j+b8D}G)LXjzB{xEh1WsHWRe zbL2=~Q6;W_Q$9w=0Rbd6J=d-Htp~mtf}@(El822^vw&(yS~1Xyk1k(bbGjD3X=SyN z92doGmS!O#vXh~JP(TAukdcrXP^)}5GlBz&S|bTD6PuPFySK?HNnzld!TpZLDhPWtZU`p9UQZ*Ue%rX@lY)(n9hf|B^}BLfIuL=UK=s8Os?uBSf?)ktyT zUO@2_qB!{Iiv2SR>$hswU4~u)k9R=Z9A0}Fi@az{a`Hatue)wrX}xb2aZl?OlzEBuZ^-`n)v$aolX^oShUL8(&&}x z={0ICRxLetuW#vn$D!%U9iZ$0BiiFTdtY;F9rK`)11=%OOs9n428M~(pV8qu=y`?A zG^4MyRK>9;i(lUpc~x?^Fv+|hi0cRN*&yM`hz%=(pQ=*2Be+X z-v5eNS+DBcmqZ#j$=)8kk)UyBcbFs|R8+Og``=g1-*flgWu<0gmK6uZ?`|WMr2?{3 zhzkCzB2EHC+VI!&C&IbRZ_;%P-QsN<1zo_3^{Epb@2o;u^f>{A8jB==e|fBVjk zGlDmKP$3Q3f;hZI~5QXD$~s54`+DD2}rr!tky+@g@Do5kZ34h^e^ zzmkC3JI!y#@2h9P^3Tp}fS!{o4ZN1G8Qzk*3eYO5NoUxFcQ^Tb)^=dD0h z-|q|^bBipg@aM*^Pot5e3|bX!R;@5aZd0f!C0)K=Rke}00@of{V5_RYP*^u*XMxUpjRfxD*L1Syn)3T0d?!PTHph@PD4;l_(K zWTY09QNgtN^!mYXa_heR7EPzRF07Ur7kjS0y192EYp{Vj^uG#n@e0!01KgLN5hjH%(XO@e>0&=CvgGcCFb%$?C~&HSZ)t76^{o?e&5S zi>ux3@i?9;HeJG*q$Se<0XzHJT3EfU!alo$TI$R@Wfqvnxv4K{#xt>BztGSRY9@RE zE13XRiarw(#?Q8{vjOcdJ+sdG)C&+@CFxdPPm{7`kV%!!oy8B<>K3u$8g!2P%plxv zyEtMY_`eFQSsrx03o=eq9Q6Fi;x&QVX7t!acdVWy$*rB-WB_~_V0zz`?aaKl00EDh zjEWvurix382$LS{%KdPMiv~GGGiJcVs@{5oPT_tW3U+1rTm~a(cKV&bBu;?l)!1YA zM_pbij86Q@R2d|yxRm4OUN-y9s!9igflVOkT*lOZB+*lIMTQys3NI-W;jd(IWPmD{ zig?0|uTy&7PsKy{YpR5mQGhfUHZgImRB3_t_YWo^F8#=oQp9BE`0d-ex3*h(}!Th2*`@!*;A`{LdHO8%f? zeafmk?>9e{oo!%qgBqYO+x?tMV`>kUJ~)-ab>b#?$szt3wkKyJUL{Mw3(O(2b(R2! zsxC!W@PiU!xBdRg)t2$cUne=~)`JnVsvctDz^a(lID|ihTV}(CkhCm#cM-eV>YSZd z2dXH;tY+U!CG)?Lk;%N56+v4*e)r78UG;MjJq>dK?Pzv+xV(aiNm1SF{-q)_rEdJONdz*@=)0rm8GFi>jVCl!JscF2pUSQc05`*d>nGQXc%3lvi(lF=BwS{XtEdS9 zTu#1{m&H8QX-I%J8P4kNZjsB)pS~wkaPzAR{9 zA+)w&4}<~&Vj}db^eZEVf}teC^vn1tfqE<$aY~e1cw^Wj42q`YRlBMB-H*g+Aw-As zwW5li87G_0lrtqv?x2P3^hI%!Qz(}Dzy;~ZGVB_G;Q)bGv7i*(!R5bvwU$Tqqj35B z-0O4!foWU{;O*bUdy@p2CcZH6VIlL;KHTFaEr;Y#$Wt6!qF$niQ7=IXxM@|!t;-HM zqrb!1UPXI**@%~oDr!xRB@K{T6V@w}aivkPv88nf-!@@*lO`1*R5P9!dN8oz5aZlE zy@GAEN4*Vxpg^Z4tbJN_Dw6V!Tk9}B`<2qLmBqMLO|3PcbOai$;394!$3!7d`%qXp z#Dcll=SV7U-A$k22hL)e5yUn4j2Fp~hIUlCTh^(szdb8ZlzoHVt?8livX1~KpPy1r z#%6ucWuc*pt%eJGEbMcFJ)n4|6QB;g{Gy$bPx;-91t84W&R=g@OjHQ0@~3d6xSb`B zpI2%_(y;mGOm*Va=A&3XmP{(Br#?#*a@S>>nva+nmgXVnO-cK?iI%G9+Pr|FCdLD;_5jJnyspccmZogw zigDsHVKF`gb$|?I{sJr3u*(+oneq+&z~K33&phPAob(rL3gw=nqAY>3=h4tV&GBd( zf9wD4w_@N{u}Mk(wb7<^+Y6JidJ!1BYoe6g>xUK#I^QR;JjE6xF;WCC+QEb5{b)%(xVV+K?}nMWOlbBN$=##15&+N3~S(u!Q2 z727(zz8YVGtK7$&5DKBgCk58e&8MPafx!y#i4+O7FKBkZw~D=?4z*kmG#Go4^XCsE z<+xc@t-q^!iu9IlxV?E51)~o!Mf|IuLi!L1bz~7P?M*$*h#plbiKc>jk1{Rb0rw)b z7z#wZI-`(w9VJp6rFv@FSV5SwUJAM?{jk$3`8lY(j0)WtYa)P#de55vD2%L3)!qaj zQdK?8w-&%Z3+Nz|f-HA!u+ea8g?w+u_Qp=~K!|A$^Wym}@zRP>F=uFtLLrVXQtW1T z<=xFlvq4;pd_fumIR83#`mo`6=4T^~L02-!G#;8XaC zA!qj#4a)euOh9zlR=b`$@?R|EtQ#1X?}8qDuf>Zta7>i!{AOz^^-p^2QP$?Dnf}hT zvN;gn9f}M=q-T8E4nawKJnF^na-;I3R6c%yLpD?%D~K@P@j5Q#BG{D}7qmhODXL)H z_RTs!m(*lis+S5q4<RyH9;3OLkJe^R+I8RU~2WmuA0uld%{XE|Cfe}EET>7R8yAh4T_!bRhTLD|R^BV8b zR3!>q!OSalkn~%*y7X!lFGL6G@Hf&= z)ZOM(=N}FDoNn44=&@ z!(E9Oc&;RP%@8+v_CrdAz*nPk`aKh^I^@BS!v6$X(Cj(;w@)r#e`i zCPC?_^G+o)LBoF_TP@mQkjU!|B{)?qwTd;&*5E=QP&9q1ey}bO4ep! zc!?mz*JrYM`RMhSGKfD2GmtJb7$|WJA93%HbNu|6r2{mU3y;{}c>UXpnWh)@1kf4n z+q|c!oR>4p5HI8X94?QmIsUFr(u(pA-H0zTN($!*E53IXj1bnu1XjgNM`uAWWrWwH zDaPwSv1&7$jp~m)ep}Dr3@vzp1%wNRE&)ywVUvDnA!obZ9el`Jp&K4;jD888Agsaj zmSBB4YI~n`&Q4W*8|C+MjJDSQAey3IfsA8WM%FginXq5h}p^tK6?$D#Vz@RPX8g# zUvpj6kUV|PaCx@&m!ZCw^Zp^%KAo%2%YVxEpJ71SzVdSsxpQVnchp|q&GmWs!%`3n z&*=+i4FGdF$6?L*8vJ&ApK;+heKGRv$6CD8vMhn)>{ave3+whM)JrY+DjRdL-hgs87LZat8CZjXZp0YrZxHwA(;9%;;ztkC>jth-_Ei>6rQh4Nc@kce4sG-yT}S1h1rMM> zjPN#kOpHzdnmBx9S*Mbtk_@A!$8jIeR&Bj0q2aAu4Fxhdr3qFRkX+6icKn^QS5P|R z*=8&uOqY^C9EwCKkJr2mpGiU$43FaQ?hTp1HOJ|L$7}|*gz6eDX+jeCI)e9d$!Iu4 zgr&OwR{C>twvC%?UK~Z`V%KLXm9sIWTaGv8nc~vuP}L{$a7ZE3u_VfGB%)l}gaJrN zkWs13Z0n2qo#nDIsoOuA_FalokPX&RT>qRXy7h0NEYyfdZM|3C74w+pbK-y{g2KcF zaG>PoK=$Hi8$wc~#XagG&Y3WrdD?23=jmC}uIG8Oep)9SX*Jaj1Ntt8M_5Yz4i4TBYx*eiBX9h@MJ^%-T21Mn2+P%+UjxFFD z#PsQnt_vR;Ddi7#3%(7_Vf0w4(D;P3gd1g2j1;3}&0eNsFQMnl()v{Ihu9vn-gjisC_K=X>Zc_hyA_ z1pyb->TgfzwCwz;e?Wv;FEu{&i?R`IHew($YB6@82_VS>9Dz?Vn8CPjtcjEA(eqYZ zkB@5DJ#aPe*QKV*v87r(h^o|psjI6BGIw^S47u$J<`U@a`dhP$`vT3&+-`*M*~XL< zyH4v-h2HCTkif3L3<_9dn1>FvzkkcWe>!I8^<9j^u#E~m*>^z+*w*LjeL6v{?OG}e zPuA0;_^?5&_D190nYmsm#O%${BL_Z*>@ zaCR;DH?dBIg~Qw~lMEm*DNlu=hmv!U;3(Po>v{`bECQbtVgx$96<=c2EniQHXZ z4iw*QXebf(NNcWqv_a$0sym5I)zSj$4bDBd-skE%@}CxZmoIV^KKfKy05^?(%9l)) zYw~-o=e@@o`lXDBY0sX#k|YO*)_<$yWm)WGy04-`avQ7&(DFVxu2utyZY|$@k*IEU z9dn7>#X*hTAM##UcF1T3YaBX=v_c0$N1U_?uKoa#c%B50qDW50ps0Gv9IW%^tK#aJNidACN7JK20Iaot24DQrh@ooi zRbdcM#MoBxFOm{d0E^-3Ma-uXvsKtdUF&m`sHmq^>d03xz4f6HEO3o3;0mzNG9sNO zf6|3Xx1BHfwA{l$%;I2b08xh;(m5%PVBzEazJl@KN|YTbfSU9mTTufvi9cMBzxR51 zvRugL%YI{`25{v18@44!N@ zRbwake+s$MaH!v}Z;T-dsgO|ptszUsF6pN!q^!*t5@Q?Lw-`*8Q6UN?%VaF2Y-4Tg z6UHxN9ZT68%m^vVm=Z(6^X>nN0vy@)fVRZ1l|E!1pb=vDyE)@a~j;dhl8yCfD{bg*PwW!Is$K~P* zcKQvj^0p)8A+iX0dA*BccLSVH3jXD@r*ftj9hCV0c?7x{f7LGv@$qMQ0oNy-I?SUj z>7ulV3h-s~Zd;~MeU>b_zHHDR^#gr-pRQkkpO5e5{nh2CgSr)gC_g0~t^Fq-o>iTE z6I5Ka+aP|7PtS0)UJCf3p5M&KW*sq*)R4WFE%};CN*0n5GP6Wy1A2oODwh2p>TUZxwCU;9IwN!h=`0wqL%rs7ZPv z@=R3bog*wjpBCIMCL;Rm)m%ew@>s{hLb`*^si2cJQS|WZ$L#Jh)HD~505bZ4RBM@A zZM|MiLUol-;3SzR#q1(mF34c`Op`V2_e8Lh0h{maw`<8yJN{RSO|34f_LH>XWJcE2 z1gLTd9I&XvuSxC816?z?waH*zD}RbB#Rq_|AJg@J;r;r$HA#HKRmp(58wo=7ATG|3eyqO zUUg``LYJQ`D&MSxj%~heR0qRgU*b%07FS%&3Xwy9;XK~hF{?gb^1hGGock`1Wj74? z^Zt$OzHiPd%AXx7mylEs;5C?NVen5nuEZE9&2vB&A=Qc~Y!OwPV~Sm!SM@@UbWLb#^)?gJt3I^shG z(Wgd*VlVjXUSFphY`PwEFL%6_cz3F+Mb+(5gL`JN`GewlN*<&wv{puhKBAVZlDD5wX5GD) z?&pDGVF z7j<0z1hJ#Y$Vpf$tde+H10Zq#8`|?iMoXH1Y_S#}H_(Y7m74fRTl{rb&IsIF(2KuBMp!~-c-lRCN-Jtl1gugs$dua z>2Zx2;ayUtZV1R68#lQvSvfCDa9Q}cH9H~x9@*N$-2q$h@j&0J4sfjV(tehUdiBW9 z6QSAsW-6}!0r*K}i$X+QQc{oSvZaoC%Yv=EBC^vaY`$u_&I_G7+Qj{fRos=vuN;G+HZacpu?sTRm@Lv4Fu-A4GDsm(j3e0ee?r%+bUH_obimndC{aKrDX9E#SK@8_DR70{5%ol{pVMs zA_!Rt29BpTk=S1Iw@nJrfAJr`v@c3Yo|Mg$C=Sj$?WHbbkB(ls&h_aE)i~5r*wcaj z8ChmlYnQjzOO?ye^6dU*jBWd-)V+Jn3~ib-RR}bz+7}yrsJg zuAYmQJr1#w(3hxi#wBMay|`@p`Wf`zA>Q}jFju88in&6F zYHOh4WQ!d6beVB?&TULGMhL#4tpX?ljYTYdSJgQf8ojH!rjzPikyaSwsn+T7wk*P1FCc*2_6j zqNPaDM$f}{j7+~t<~4c9dtjEfnJ0ULMO8$U^0bNZt`w7ziIEQuqQp{g=Xk1g-PTr( zkKaSwhPZmfR+?oxo}dsMW5z)~HtQXo7)$r7DwrVkSz1D0G*em-g5Ya|w0sQ`7`>H% z-y5clU(I_=4*)a2Lf3~ZB_Vp#;uYdeAs%Kxwo9pOD=Q(Ah$IZ&ylp?EEY`NlqEGLA zprlaTZTQ@F!?_wZPqzWSrLw;DjBV~ilK2a-EEOgj9XifOtL&`e7rVe@$wuZ5Q2vm= zD68TlIGW)GL7`nVNg0&r z^TQ-f5+BX40Kb;|Qh9%J$F_8*$8<9M>)zCbg$E_q^t4|ry06W(3bml+W#mU^Kbalu z@E30&YI`;-LP!uIoSK}8-^<2JpJ&%gQf4?!ySz+&W`|pB8aejE45n!}wWa0frPTfa z@>9Vptw#z3#44kA*0k!iyVf4F`ON=n;Z-UOJOUkLPEu##QF<2T&G}uTW7sPix7c$j zteA;oR(nG10?NGS6OtBCeEIqykQ>AP+2I_}e#yLlM>u|786jh9vZKq}*vM0DMHs00 z&(sOe3S<&hqY&@kN1ujBB`f#K|7$KL7`Xprw_u-tos(9(QEd5bT{x8p+#&yB!tbAo z4g|g2SB7uhL{r=#e?0Z%BDl8o*w&-CxG8fnaxpe?e&MK!&(k!n<{P_G#*;f>U5|_Yr_00lebiC$O z2d@aZRUt+S^JVW< zHmb2!)+dYr?TG1C-3$U;_s8K8!o}QIZ4C*7%I)0(z|M&oL7<5m2Nvg)dCALohRlB! zLrcZPpkEyKg5Hz2x3zP(=}smjkGc8y#LvR3ycVkfBJ(6QLqU{R_pB?lcurc^Iwm{S zh9u6p=Hw#)Q+wB>GX=AYR$YH%s&}E(Ewjk2{sg_Dqd7Ea`PMLz`wo!Wcvo6>A>fmW~Jlqs%nh)0X$$TVjLpboRkxT)U^S#8?)D@I(vE=rgxBoN=QJ5-;8n7cmlw*#N* zWF$G&2RP9+cYlBGQ?LJJ!%QN_a-q;eU@{I9q6qteR1_t?z3cm+N-bh*iWx)fK9iW5 zT59*W2d=E!%Z<8no0>8f#q^jB7iRZ|T4|DoK(PX&U`&?biv&=%&&qhAby)bnAB0Ww zuEDC@azEscEyD%1bYHx+vdPA(p|&?K&(@l3EG%sjruWP-2S?cu=1{+gWM~|GTw8#H4-<^;c@}{;7VrpEYMq5Q3d36ZBjKD0}_u5r95Em zhdV$4FG;$UO-+VNdB;CWO6tr9g3kZ9&c16K>Z76KRz3HB7@*1@2KU{J?Dy4QykqI` zPh|gE$j_>K9M3TahjHri^_;0T$aq(>^rC9@;tPzvCuC8zaWRV%qDt&jmXR4V2W%YX zR{b(%O!u`A`>^foPe;qhY&(SYY{>0^NOg-bXVNi|XNyHIu?QKNcQ=67PLkOw+_P4b zl!xBR$4g%znZ6#(7^3OqOr;I1-hpE2w9^6{#er2*bhzH1S+TJ3`)GtO+89a_|FE_~ zTdmhvH^w_ek4l8nD#mEe8l)?(h6mRP^a#bmPgONgHyVU+gDGgFAlr;X{JuM-U0VOe#4SZ*769;Gmeql}0k@PIPF;NvL z5t7ft;%(q6&?KpTEN84+6ctt*e}w#WYDQulW~1RiVB05$BgLLF zvlZP6V|rp3N2X2E4YSsK(JMG(?(nAF0UoP>*6fJ-btq*}e>b^)X@*kFUI8)71UzY+ zBO4q0hnl8l8bQWJk4)HcxIhx5bjKZ!B+m~wc-n0F-HiP_6ML%dwEXtjJ67pG=yTo--WbD|4>#}*;%t6q?py;1!>#_X zq*I|9FCWL?YJ)IY8LUgNC-iL8qxeiTf=>(N?KaxR+Bc?5>626Hmjak)jM&PT z&uy$2fo;inM&SrWqo!cHA<(jR%Ly3fW6b3&;L@#DjbG!oY~IZc6{4#7D#6`r-*NA{ z>Bcf>2+)DRpIUMnEU)AwbXRV8iM6|d5!cgJ^LOI&+{RsS(_FAOi>UO{4U(=A zx~p)Kg;!BqBj@NH_@U6)LkV9B`qz|$96cfjD)=YC7ZKlraM88ibQSnhYB}FzdgBa) zkb2JIA9O=}p|#Nxi*XoJ&>Qys!+NyMV($Dz+#PiI6O`GeWqdxuSI-af!)N|G-e*HdOw*-mp+S>C%bHD{Wtc?`i!YVD^M(KSP49d1fQbq(!O@=G^py<00hcQ{~*!Q{67?{)u-%cWyep&yw&z^G0x^Q)FHAX6OwKk>pnRR910 diff --git a/components/outlier-detection/seq2seq-lstm/model.py b/components/outlier-detection/seq2seq-lstm/model.py deleted file mode 100644 index 6ff072b47f..0000000000 --- a/components/outlier-detection/seq2seq-lstm/model.py +++ /dev/null @@ -1,93 +0,0 @@ -from keras.layers import Input, LSTM, Dense, Bidirectional, Concatenate -from keras.models import Model -from keras.optimizers import Adam -import numpy as np - -def model(n_features, encoder_dim = [20], decoder_dim = [20], dropout=0., learning_rate=.001, - loss='mean_squared_error', output_activation='sigmoid'): - """ Build seq2seq model. - - Arguments: - - n_features (int): number of features in the data - - encoder_dim (list): list with number of units per encoder layer - - decoder_dim (list): list with number of units per decoder layer - - dropout (float): dropout for LSTM units - - learning_rate (float): learning rate used during training - - loss (str): loss function used - - output_activation (str): activation type for the dense output layer in the decoder - """ - - enc_dim = len(encoder_dim) - dec_dim = len(decoder_dim) - - # seq2seq = encoder + decoder - # encoder - encoder_hidden = encoder_inputs = Input(shape=(None, n_features), name='encoder_input') - - # add encoder hidden layers - encoder_lstm = [] - for i in range(enc_dim-1): - encoder_lstm.append(Bidirectional(LSTM(encoder_dim[i], dropout=dropout, - return_sequences=True,name='encoder_lstm_' + str(i)))) - encoder_hidden = encoder_lstm[i](encoder_hidden) - - encoder_lstm.append(Bidirectional(LSTM(encoder_dim[-1], dropout=dropout, return_state=True, - name='encoder_lstm_' + str(enc_dim-1)))) - encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm[-1](encoder_hidden) - - # only need to keep encoder states - state_h = Concatenate()([forward_h, backward_h]) - state_c = Concatenate()([forward_c, backward_c]) - encoder_states = [state_h, state_c] - - # decoder - decoder_hidden = decoder_inputs = Input(shape=(None, n_features), name='decoder_input') - - # add decoder hidden layers - # check if dimensions are correct - dim_check = [(idx,dim) for idx,dim in enumerate(decoder_dim) if dim!=encoder_dim[-1]*2] - if len(dim_check)>0: - raise ValueError('\nDecoder (layer,units) {0} is not compatible with encoder hidden ' \ - 'states. Units should be equal to {1}'.format(dim_check,encoder_dim[-1]*2)) - - # initialise decoder states with encoder states - decoder_lstm = [] - for i in range(dec_dim): - decoder_lstm.append(LSTM(decoder_dim[i], dropout=dropout, return_sequences=True, - return_state=True, name='decoder_lstm_' + str(i))) - decoder_hidden, _, _ = decoder_lstm[i](decoder_hidden, initial_state=encoder_states) - - # add linear layer on top of LSTM - decoder_dense = Dense(n_features, activation=output_activation, name='dense_output') - decoder_outputs = decoder_dense(decoder_hidden) - - # define seq2seq model - model = Model([encoder_inputs, decoder_inputs], decoder_outputs) - optimizer = Adam(lr=learning_rate) - model.compile(optimizer=optimizer, loss=loss) - - # define encoder model returning encoder states - encoder_model = Model(encoder_inputs, encoder_states * dec_dim) - - # define decoder model - # need state inputs for each LSTM layer - decoder_states_inputs = [] - for i in range(dec_dim): - decoder_state_input_h = Input(shape=(decoder_dim[i],), name='decoder_state_input_h_' + str(i)) - decoder_state_input_c = Input(shape=(decoder_dim[i],), name='decoder_state_input_c_' + str(i)) - decoder_states_inputs.append([decoder_state_input_h, decoder_state_input_c]) - decoder_states_inputs = [state for states in decoder_states_inputs for state in states] - - decoder_inference = decoder_inputs - decoder_states = [] - for i in range(dec_dim): - decoder_inference, state_h, state_c = decoder_lstm[i](decoder_inference, - initial_state=decoder_states_inputs[2*i:2*i+2]) - decoder_states.append([state_h,state_c]) - decoder_states = [state for states in decoder_states for state in states] - - decoder_outputs = decoder_dense(decoder_inference) - decoder_model = Model([decoder_inputs] + decoder_states_inputs, - [decoder_outputs] + decoder_states) - - return model, encoder_model, decoder_model \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/models/.keep b/components/outlier-detection/seq2seq-lstm/models/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/seq2seq-lstm/models/preprocess_seq2seq.pickle b/components/outlier-detection/seq2seq-lstm/models/preprocess_seq2seq.pickle deleted file mode 100644 index fd41523e06d12ea68a7239b0cc6345113e327e64..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 245 zcmZo*jxA)+h+t!2V93qP%T25(WQ;9j@|xeq$dKsG;LX@p$ef&4np;q*mz-aes+U`u zQ<9ljRFqiB6m6JC~8y4@8{>|^&bepgf~M; zp+HinGl$lEefI7cnFED_C51vs87vuK%Y-w4Hi-lkib8D?%isg5jqR6RKl`q*L!o#{ Lp+pkMl2kna^z=;+ diff --git a/components/outlier-detection/seq2seq-lstm/models/seq2seq.pickle b/components/outlier-detection/seq2seq-lstm/models/seq2seq.pickle deleted file mode 100644 index 5340cc328d20f107bf84416eae7df6b10b05b2d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 38 tcmZo*jxA)+@b2+uj4fpJ7DzeeEF8DIC^Fmm#cK&*u z_3Bz@2}?WcOQO5m_4u21-Rm*nVP}0oC-}?nU+{F5bgyd}p%Q_t{8+IvLXv{jy}&!l zFLZXWcaY!W-&-mPh>`zMFWkAnCA#`|L%(3(kU4@Lfx!#I!-Rx<0_OyV`uT<&OXz--$q2c0;gS0t&wRg7 z@5Qqt{6ZHCNEg)q89p&V8^Q3ur%n~|W8}Y2!h%tLjQ6wNaPMMZY*&{~y&?3+dgJd= z|0?+(#DQ+T;oddpuB9g=)Gdfz<=yLj;{PSybnA_VA4)=y+Qd8If5?xlKz`JPBK-pW z1H%48arDw=yToItkd?HM)%ULZWVasa#PZj_|KH@Nt#4=FmaYkpbeb^s_scFc-F)bL zAP^D~Xr09&p>r36bm`fy+q%&2ax6e)X(sshSL7W0q4a*GUmW#erWO_{D)= z9Qeh7UmW#erWO_{D+$QXH5xe!_5Jfia}p&UU5CF4AoWaOvGiZa|maz^;q> zm$sc3-R-VS>CAUb>x3@d?Yayv-R;yvJ1gu2M$aF~uoDPP{0X1kq|S;ZUFF^O8;kJH z@@{t_+I|S|Ik^`a<3Dj%9?hg**KhvcLN;ezrsY z@L$*qhU~UGc9|~)-n`EA6P@OBLqa3HL+8xy_G@%HBg_TvfndJ{_7;Cd{6E=K;0*~6 z4fP8SoBbCY|71^}K<~wWGveR5SImESKmIv#e;)IXdj37R|7_2HK<@9{`Xjq;r$4oO zVF-6=P+^fy4c4v4CU$ZB=e(#u)Bh;zrLFb5A4fY?2ze=k?;vR3~n%$qrQhLBLV2L8SJKmF&A{C*ELJBOCk>4bl#pA+=D z+W)6h{A+;!iX8Y^KihQ?Y1p-Xx@rE$dQPBW{;@{+U6y|I)$T^{6YPBc67G^BA)$t@ zb^Ak}ex#q*acAkT-=8_~M|$98|IU^QUGn{BdZ0zWvwipd`R8$d<^Owg;E(ixMc4Z2 zp8a1QO1cXE{{G)t(S3LCmS^p51V6z}Js|QIJ>fFAvq5*ig@@6-~IlK?U8fr0WuPaQ@ z$)GD#R>7_@U*KdxC8~@&hfX(TK*jheolyLMameN9kOh;m&G!pD)4m3*j4*FJDFRwc z6nK;CMBQg2<!aMjO?{ze%?nr=y!fJLU(aY zbSPvQJF*&YQdl*o9JX!zde$P(kM)q7#uAA&>?}H$ZF_6W-j8x+2b$Qih6nsuk#ouH zk)0FRg<`&}kIYnd+i(8th4#7ZdXo`s&A2@Fs)955ek%&LiOj?76}^Go_Q9bwTI3p_&B;jGZAdcWU}!u&=<)_2DXrm?(^8hu$u zCvi>O&|X{dN$w{a5qp@_1`fgX!_HDZP6Kidu7hzUqHM)W1tOZ72IrpzvY8j3(2M@g z?CK{Aan9ZqaNwyCuBi23?=h9=erFw-V>Sdvnq5XM)s;-21v6QRrOD*dx-sm-tXmM0 z(Sy>LJd_%*rKRT^Aak}1+bX>s=iiI~Qym$0=9MN`pr45yFSZi(HRaR~twF^33@Rxs z#AhFeV{-IJyf^kJD!v^6jXu{LRHbC-VR95%`7>}nNf$F6E%>-&i;2pL+Yps#$j)ib zM2}rUXkAvs@ApWjdgBUU5BC%ncqzbxqc2G5oE-k`!WDvfTk%%!)L3on$5`KR74}Ev zl5746{BaX2I@xz0w8l+l`;v4Ra5|kI7$47zd-P#r4r!rKl@j~KDh`TX)$qF_d$LBK z8MZysgccoH0oM7G$i=3!Wc8kUW=%=~XpLSEy%rB;leX!Bwc;vfT=X&EeX@wsjdf5~ zIEmdo?iNlP*#|eYS+n_xYamd4KgrS>!+!mg2f{1o!tuxD>|5(q?Bwms*qlUPREkIaX}T`uE|uS{jI%~1 zLI0q)SWW>Kw3-(@xf^*$8hco8Z6&JHR--KaLGsNt~v{VWNgJjOcNY*^sZu zvSW?#&igm`LHZaR^KFK7FF}4@e`FH3!&k3Jw$*~cl_pEzWc3i-dFLRC*AX&PNfp-{ zZ-oAg3gk`Ca;RFj5FS}*K!R~!@Q+U;Gp2B4n%zWDwKzsiO)oH?kF?SCW>s*fk>XbM zWXc?(-qiluoT>mwxj3_ zQw)3}&w7k`PsRt1$7f!nU`WV&c+tZjWZSaq`W1#?{Ma23v+@F(Z>Csza3{Rqoo3J3 zH^E%vHcl_?J=NH!g*LS^==b?Ky*62im3*lNljTG)9OT%l>UW^I=qvH;JD(hn$b%eN z2|i=92+H5r2EFweARi&PCzOu@wTfE0M|vM#zS;u2UGG8dXn%%zvY0mB8pm=6k}>po zG$|OaNS^6^!GjXA?8Z-xWZWiYzHiMcVsj#q-knxk|3=!2H`%L6`@9{6cSI`LfM?bX zrnd~~x1?k^IOi~Wf4GgcXHD32#~XP6=oVTcE{3KN*6h1$rVXn{;e5eiD$__uv`K=YlW^xydq(H*zn{Q~>5ok_WixNbYs&-woY zJ5%?ocbDg(bD)2}o#|)0Tjv{Mw|%sm58d!zpTE0N=Zyc%&bH*=xjY2R;%9zzng{>$ z?U32V{~Uk*YuF)wesB9T`-7xS=j?vjAO5Y&``xNl`j=VW8M=KS;DxtpKaUH(hztj|w=<4Tzen$CT%k;cKe?~72+z3n#Sf6*BOqsiiSa?;DnNQ}!v|KS-E*0l+$Uzal>8{2rfwOV&OFO3OZWYO!ag;N@ z=YVgjv!JZ44l3RV*GG)m1X9~$L37l6;D6HxxONOBnci%mdr#JG>vCLY?*liI-0@uR zBV_HC7`(qRm&tNDO_tn0jid7SLi)9QH2s`Rh4&=G&P)X;Id_^`_VFOg_HQ89BUC|b z5Rap7#8b{!f)~oRqHm&}kS*MG?$W+b^p%`F4n9#$d%w=1-wx|zfU7i|-6}_PybIuq zHjj2&H;~EsYr$Eyj!uXRCuyskVYK@pM)ueuy1Z}zJ`w5#-vfQ1!D2I!;%DNOE6T)o zeI_|@`#k-4Evs(I`S?J8?Ms(umK`uY6(IytT0Mqq`O z4Ev;R4ko2-1JMh?*#BVyy=JRH`|ah~X3v!xg+suVChoZXRmQ+P8TrH6AS~7!U4q78zl-vi z0-mTN7fToV&ShR*@_-p{&8ha`f%xF94Z2QFB&SP%BO{}YxwckmQW~#HD*~ipnA$5` zmO`O1Nd{Ub+$RAabI6AMmgrUCOLI16K$HGzAn$uXdDdEdk@pPe7?g0euKMt)p_&{x z6a$XW>Iu(Y;kIo%;qZOj82B>wIZd3ULP}ypp>XPPP(K|GdrAQ!8^^;Uc`Hb&7bPhN z&*S*f#V~8xEl6J~P3Au#(BPAbZkJ@?{2V!|_|O9TZQ2IUH@Bd}nH-4Ne}GOYvF37e zPl8HE3D;KCL`yt8aI%Rg^U(bp%{bzM6)xpC)e>>p$v50bzL-FSE}l8CpRTP!Zim7( z(#Hp>$QZzcKt1l-`vYW1um*cYtQfyE8$th}CiGrXf9w@1#_k`rk$eu$V#WtaklLg3 z(BJMQNtfFQ#o`T|ZrWz;t1g9hyFY)?93s07V^wroi4WJfyDQvwZcC*y=_f9}9L zRTO*8Q%6M=IKkXzW@Wg*%-3c0nWN0HXJ9k=s8mDxUK;@RF>0uAbA0{xQAc5+d2eLj z7LqmT4xsu{1&+tX&^i4Q1bS);Y<^Kp2HkWZWA#-*Muw$FbYBn!-&nHeSRAIB_M@#f zj>I>86<2=U9Y-`dGtM#g%#G}Kv`*_geOK>Aq^E6Sgr+It*2rjVudSy&o+#C8eeou% zMBWe^D{XKq*-cLvh@!^eQf^oCY;e4o#p!)7Ccy^J={+4|%sbY9NJ*F~t*(?p=S>Hp zM`J9csfaPh=Co3&r)QzcZ$9>aAVqrAt;H49k+@mHmrRlxfQL(RxWH?h>CzEPaZvG1 z{4Bnmv|mYrjU*^4QHAt@RLAxLJ-=%!Nps%MY2N=rRygTldQl%dJ-M2m%aemy%{@T) zOdLIa=`-_a>mE)aG(ccC4Z=GgCehI8c35nhNwz;x#~3{yyqY8miR(p3YFq;}Pgg~! zq~|1hSSY%m+Kge&SGekzV@ZUl2kdv#Li4Y6g0(3NE#p6v@fDxAjLoa5-NrN2*UlE6 zAI~IphbP0rr#yKgGYBV-8i@f9U1+rA3VPS-6fGIDoT*!)i)mwKV9|wH(6R6)xv*>! zxJRndh3m&s(d%c)4rwQh!lU%0%0-H9p}5P=jpk}KG5so1@rlC`l8~JNH_O^Mu@!BM zW5;!FiSu)EHFzz2D!PTa6OTfk@lf{UT5IU&uSWDcWU(l}l*D;{CU#jGjQt)bm{k{w z9xtarQt);x+j^0#NM8)D+Z4!P^u?uvl?iiZAC;da!g*d@h04w)IAOUeT@hqNKYKr= zsiLz`DzO0f+*0ALl=s8KuO%@5;C`$exE5}BSI`|Nu0qG>A}&%g34U8`51X~6C|=Hm z3bG1b*{G0yl@^#WZ9FbJZ9r|MdxG|zN^nsTM%fLT`0R`vD2OY9KGCiBY>z_ckw4{NB2Qb#VT4a`soth}WS*zb?l}OOA8fFoCw-Sc%h0e2Kl16~6fVhG?W%*Pc9Xwi1?XCg_pUnZ3w(M0DP zx6!%@bK$6#A=K{egSW&JNc|QQQt)O7thu)WK4^%OvcvVzTdEZU`m18Eqs};>@;(hY zSLyIVcq(o7D&f+H_rvCrW6bN!Dey$|7PM8|rn@fcgJ#1ll5Y|LF+-D4U41ppebon& z^DW6`rhyim5Qn9!s_3IqOUV4BKrU#d(|(*S+;1F3qQ~B(!C}Xk+VjCQ$8!-ih;?LM zjP6a^U>&VJnlr?I-CiQzlOy`D8^@Y|qByUV%(psE>b2C-F_%E|@-RC4*e%BH{ty!K zW)+A^490QdPK;u#CQQ;^Km%9{XwD7QK7kpIO<>sK3}g(s)jfdCP)oGy|qBcyYb{LR|oFP zY{>~PY3}CUwe(rc1b7?mN28K0$e@W+$&$JK(YT)$%suvqNZW~$8)gi7oi-Cj_YR@+ z3m_p9FZj!Pv;@0Nb}rA_ zJ*Vo^`8#d#@J$PtvRIBPXc=PDu6j5jR?KX7qV&O?8O-}QCk#H(n>0>gFyZ7LNS`N; zW>vM2SYb&VHfLdueHCP-Oohj+CJjs3gbJf%!RCN0xE+j#(c5jHNhBZ3YXU(@ZX`5q z^T2D9H!*KNS%B5yIL1L*1m{W>Q}>(Uf-inVcVF$m*lj&1l@8a1`px+PN}CNwDqZLHlJU zk@vU1z=WLP_~~GT!{u@tkoVlr8KkZ!U!2tGq9l2obL||qZ|=ouKCPsB-z7+3ln1E$ zk3@rw0=;A21VtlTN!C8PBq{H-_SEK#E#vRCTw5)}|J zUJZ-4Qljb^MuXdCLa&5mGMtlw_)tmK`#}&q?P~z8$4khID=K9ErdyrV(0>8Z1{ZGX48gzcsafoG<^q%7QO>DSEs_@*{P6qcpDhSjUtb7DVch@ z8gXSQSlMZ^dQ;qKv1A;)QdGytZ^hi+y^{3q;v5?M(4X9BPsM90O>o}B00+5Fg}!%{ znSC7+5N~OU8)+sQH)*1ryCG_*6=TjG1wOf^3Wt|{qegwyY09m5I@?GXabX<}HJpc2 zC(7YP7dsrn7n0lYhe+Dlco?{)3@f8!VREE2NJuNu{a$Mzl(UAMT_T+G@LlwoK`agZ zupSj)6ja*eg7q*99C<&DyZ@Uvc{C4BT&(k2`i3EO< zltsDZ%S`C3l@J|w21cJqu6r^CpkK;cvSC&{toj}fy+f05_GnLcyQ9p=1v6{z(MU@)azSs%q+JcN6%>E73Ybd@mnhyqr8+<-4y7? zjYy_2EY86!sy_G%vB(#97?}7R*RLy}N@fW(iyo)7eQaT)tUQ(r4aH{Dd}M|nA_j_4 zQ2Jbo-Jt5@uro0ih4i1$HHV3H)VOUbL3+jqF(``s!iw`&NQmh4Ap=_uxjv>XgevZrCiZM524hUq`En!Gx`hORW1 zCg+sylY0lYQ7z{hhY0iOf|!9d*!J3pIL7azBUAq%D06%tOw zp{J1&ME&+o5CNb7Cm*dtL(@Q7@G21MzXqXkKsEMX@`S`nw9^JjZ%`gIhV)#p7iQ?! z60w;Co-g}Gr_A31WyX^rcHmC#bm&7mvwQ=(Z`DJkNNu1?&QTc*V;rve!_dVep;ye? z`jM56sFUIbBML4t`#k00(VHShC}1VG9>nm(3qz0^IFYkW^oK7_K3LUSMGY(nd`nZqj?g zDhvFBi*wOU?h&;*JQS2&zmN?B_mb(pV?ptN99I!E7NTkvkn#m}jH=aSj4VmyGHZ4d zW|=w8;7ssMa5#M!o{2+5bLgm!7@AY_mVUgbP7Cq}QT6Rcq+j11H1DG%RT?A=j+DzQ##9DX%}SYK^2~mlL5a^&l;s*9$CcnC9qoLStGd+6%$F&m0qKc?Xa7d-tCTk_oG}PLURhD!i!@EQRq$(3~}AUlx$apMn@TkPdSIEZP8_BoaHzOZ}f!HkQQ>Ibu`Yn zTto)HJw(kW)eGWV9I1Q0IMqC|65jOAtRJ&w937Y7h|c^#;?m>;o=^MW*{uY>m%gSC zGE-@<0sWXmPqoQpt#4#Y&q`wAvza`t^RQPudV!t@d`gCz3&ESCAvjsV7a|`gP`w+G zb>pkoGp@~fB&zB()axmuI+)?cC?Qg=W=f`gtAM30(_phn71Np&NIbU~z|&79a67A* zuFBUU+nYAPZigA9twSTdJGeUQ_*AY0f9`8!{_71!k%R^7&lLdKOB9Z z6xaq}^{gUBu4f?V1uH@Mn|3-w;~O1xQ4?)#`f`Qx-{9^b6C!PVnI1ioh4;K)FlNkn zI-nqiIgy_RBW?OX=+GhL{O$@asIr~x3|mD?B}T&I@ZX4K(GYq;yMx&o=t<17;z417 zIZlb@&@+!CDm{!zy-AJ3?G>spe}fQcG)cg~8REFjMv)wM6T?&O12NPtncC=uk@KS$ z&;+Ybbj-eKxTF=x%8V=~VWlXV4}DA2)N9EV;rYZn%!Q8dUyJ+v{isMvI8Cg#L+#RD zWd4`AOxS1@Y;39v+YT`6o1z+9_fzam}jQX7&%)4O)BrP+QOBImb{o*1?kzAmt$?UKly ztNBb<8@i)SVqa#QzY;gK#ST&(%IdY>E25KzA_>w`q6x8bV7@VfN^XyYw5wleskL_+D4dbm+v1bo+hEEr~+l+`siAR=LlBk07^MkO}D-TCs-$^eyWx`^m zc!$XF2AExb4Gm}wiP)hFk9(=%C`~~OR<HChIe0I!n>u(VBIzmO*D4Wo>A|JrdJ&9IN^nL8Tv5gdn(STF2_IfZ~q+c@N@oO z_y4PVbh%*uhsHl#G3lKDf9?2(PCx#ybLYC&>M!sU2fFuN{=AR*|JC_#zt-nZe~@3< zzu~|i#UH45#Q}6T{=Xf6&@C4~pBM1|N-q8={$XEO=Zs^!Caykj54Aj5fMX_Y19LZZ6B;d`pETHH# zo}a#h-rl*9ewP`6^Tw>lSg9!57M?^d-natM`{dCyHJ02JosF~hrh~?AO>%SQbIcKK zq6XEwFzuTmY3q=LG#xicAG!vr$E!P3j5|r@t-b)PAg+C?b~_R)EmF9~1D5n(jjw4u zOgk+CB69KgFg^p4I@;-D8V8C6B_zY%3DO6pJ5=_(1%4-ola<;1Ve!cRbY9+7FdD2+ zS2kYb&IMnG&k6TA)^QwYg&rmSSE=*iCQs>up;h>JsW)C)ZicNPZg|gbB7gtQCED`a zM>@J>9sk{18ytenP|f}=!r_$|7Vd~eBQHb62Mq|b>;r?9V#w=&abWi#7sg)cMQ2k3 znEm7=341>RCLUdDKQbzcEMMSh*be-dFG=EVHFC?2?88BdW8wSG zVPs-n3wb&DAvx-{6oRFb;M;f|Sl44CywDNDXKPK#m=DU>uQISn!CY1Sq#3r4kY`&x|6BRia4}kAOsc7BEj-sNtRn4L|TZ#oPCYl`7vU& z!`zqDZ5G4yQz}9Pn zq1U|_GDo(Wx!jftIkW4iy{{DYw~Bz>FK^>+_O?U&167{Qw}4l7O<}^WJnB=F4iopx zz`|)Wc$>75&@ko*Tw#@X$@^w>y0AJ{?AGPw+78fwtsKs{avA+MyTh4F&afj5AWY!* z6MDRwi=MlpK2TZ_mcQXi#a6+)NQW8T_&%8o5mm!2vmesxVI?sBoG?bY9YCS)R_NzC z2=j~y)}YQc{l+ks7g{lUuV84Rh_ympNzNF<@u|FR^yEL9GEj# z23pxUL?de`3=7?lk0v#fBr!SmxW5Bl3%CITE-u17m#=UmJ(BR#CUy1@U4T@7FpamC zW&<4yh>rC%`2KVnn<0p=&)@L~Mqg|MwHM#XL;fQvDih;fDw1KuQ5Tx}Y&R*MZbEwX zh{X?U+R5a|)x`Wl8ahAEA@Lp^BqG(CFT|C2-2V_9>$9Jl?=hfWPARB5Kp#it#!|@- z3Md>%xC<^-)N9zKdgFDg=&PuC00wbH^u%nK4*7Zvm8w_lN9jhUk*Fn$bU&OIp(a zD?Ig(-Tw(>G-PmC;dUHgkph~ru4LnIOL}DCItNX)GRQr+kbJ*j3p$JZm;l#>@H%5T zNE>K@$p=ZgRwoCv9may6!+C+uS%GgG2*kA;;P{MVP-WCiR(t@o+;yMXciw_{JUPU4 ztQf>)tj~nmvIk)Ht5s}burkoYrtqwl!fXvuddVRh2CukF1z*>jxEO8yHk0888K=Q@fL5NeTe1H zccT9{S-#H0fERyg0~_kBKxKtH@1ZM-Nyic)X@MVa5cnF`UeAS*wQp#AofHgQt_S7I zW-$gmU*m_GwnqXf;VOp*88@{wvgK6RC$gs+@Q0b^f`sy2k ze9?Kj-aQ=LPNm@J4Rv(d1Z(KgumgL2@5v`#xxxgj+W>v^^!Zm0AA_&QN}T65h?m#w z#VZ6X!c9w8;$ov07nb z9c2EC^|-`Hjjvf8iY^;AQQKKP_y>n87^$^)FgN`M`c=OpAH`Z}qnicMde;WLhbnr7 z4#0JfPolzwm#FvnC1|buj_YU&#!Lx^(boonvTPr|x2zqMY!F56au^2klB}&x6s^#|cV6N}DgrSN(BD_nG^fR=01lfg zjnmIEL`-%w6jfQUBYY}oed<&Q-28>;C9i_qdui~vz>>|VSc9=WDltM@lFbkB$CJLz zIIhf}{S@<>_TSY;A3NS6rWcpNm8_ilS`&edEb1ujbu|H=o;n1dABccZk_4+f!x2_z zg@V=^Kd!i75%+1=IP&FaGPG#jqHE5cf~Z9vz`3Tmjpb84vNz zRD%8O{o&=*6zsD|7bnWKQfnh=K15ZO2Hq+Gz4s-ghh_?7BqiW^?@v^G%urDMdJf_) z@mLS2Pczw~JL~E}Z+rw@w^idWCNm&>8aw!?NR&634Z;wztYk!dcIEbCu z-hi5e4B1)i5We_XfB16i8+R*Gf)(!j7^f93!`nqNZ1%9bGy+C~^V7cUw$Q$8{HYH3 zD5MI$YJEvBp%rxL<_Laa+ZR~x^@@%P=!LH3-!ai~6dzt53;kzhAx;;+< zo{cQw-SXl|Dph6s>lbp8bEQenyf4i9d#!aY6IP&hbp>9Y@Qox^=^=y7^ptE0oOAum zMfOvJ(W%FAWuOZg`M?mLR+~d*`vsCQ+=AcZw24kkmd4``Ex=Fg6zz9zGA7tghnIId z&~?HZDk1QRrT9vCKXL<}Y}tTbqYH3kkvr)fF@}ScwZnbi@bT(G*9W&*}1EsnO(>^;&pm(TJfN zGtno_7Z=5T!o02Vu&XAYR3%FDWjcx2^S}nGcE^qnvzw0j_j2&)fObrsEeUsKvas#; za9HIN#uyl65NFk`SZJ;RpGT*`@t16^kElEj+&Bdq&&E(W;)Sy{j>7SCedsiuT{zxv z2y)v^>4DoC5Hzk8K01FV3&!|StKt6S_#Q z^J$3cC0Zgn86N(ofsUFsAp6k?La-NakQWWgaqEchCS6`ss|i9yic#*`N?zOWCK?Hi zBu9%J!6fPhRoR_R>Y@kmZ!7lUqWKd1JbnY#rA>hw;_`f8)IMT9=p@}SMVR*vy$At^ z2Y|BjMBZz#DS5xzhHPIpfY%Dj!DDA$)9CYAXnx0u^qt%bG;Q4>;@km2+~sQ$T(b&k z$yk^v>_aNQOrTo^EJdNDc484=hC)splU*vtR_u+YdROP8x1vYE!bJvo%Cy19`C z?>PnG8fEa{MIo*8J;C@~PDLR>yr59(T=LDq7%bCf(^Hc**sLYb=(v-vgpt_Ixmh&R zmb^*Ozef~=dGzC_i7K->A(3$TrZHbII}si_7NJ^SW4_V(4vva2;g(6svaT~`@DtZO zAxUz*=!Z4BUkOus0PHi* zafOK&xtxkY{EG?>lOlwn*M1weyeS1Qu3klkDy`vT^_=+SGqyvUj11}yNW}{?1b?F* zy1aerO|U*4PqZeL;|GWLs1lP5>4&SauyzSwB>arqZjIrO8|lsw zg{)oU8o2XfG&JyUAULQBlNCm@>`M>6;o)$;lJRHjKO92ayIJh3n5q1hlbiY5-XE!s z??~c^mV9FNZcr1=;+rK7GsoUc#GZSu5|wMoF#VV~Up;X**{^zyR!_<%@wfD-OQ0e@ zB;Ox)f89iPToHmjpEU5}PF(=WNYodKAN>;>Kmyl4im)o?(G|l8%lD? z*92ky)fW+@8-3~G5y$9o!%TX|%Z)2eaR8eKZNx|;6(YTj$?)a#p|Fyt3;a|u_`_s$ z{j39it{HTOMFz+0U}&n`7NU4aTo9X+50@H_;02$-(6F-*=Vp(Eb5FC$O5#kHTp5Iy zrmA6@;62K7Ntr{x92xW{XNkuNQ@mTZ0aiUdNt%ME3-(tI!TdfU^cgpqj`81zLvNRp z7g;4_)$zq-i;pOKe9JRkYX8Y zxV&K&)OhQ%;bZ0q{CcY~E~o=se2>(BIq<&T)oC^8YnKqIvVP1lr?1ekHXB-JD^P21 zWBR?s72D3Zg6+l0XfV`}^tLF&sc)Bq-lEk|EU7>ls>Xg?ln;5_E7YH|pN>ABM6!Ria*pT{y~t-sme`UrUzSp{uLhd^a5|UyRPPd(tNpcHw5_x18mu z zRzeI%H39dU1BJ2#R%zZ*a9O0wyNUE**)?GfL2lbv7V#f_Kx5! zlSad%Bguk2BWcJ4&4;3-8*G5X-9$6NeI3 z?%Z~8ns^ZUM-K#*_I2>^>UQv(vkkJ}=&>_9BKh3OG2~>7GF#~}AGC!9`?4!@?JqAh z#9h1I;VAV+>U8A`ihn&rzduTcdi|RyS}F@Bez~}%;uXG~De%#*5ZDhEYV%X|Zc$#6 zAu%_U`LM{@=wZ16=Y`l|&4=L-H1`|1v&o9NTRIBLFCBMi5fj6h{$}7DYm1+oavfd` z+lx;>>cWfZO?0)NwIC*Z52$^-PK8>MFy-5FNb(jT?{ll+YlST*)7uPYecsC@eojJX zrH5qW$Uao(N)Yt>cBX!|=^{AqT1eLQKSs{B*70ONZF#GJI`pM#`35#jttxB<7_I zzw~xJ-dxu}_J0IE%Pks~2z2YKmxK5@eiCe3cprA8X$&8E=K(I;BMZH}1Toh~wvmK0 zc^Kue89xtO2djGT;AL(N$bJ`$A8zl2DIM1ESl$zY%bnrPon$g#ivt^yA&9Av%mvZq z&TyuKqWPEiSifpS+u%dRd5e!8hv~^XB3?vrs;=q6x&>=Rx?J^}O(< z^EkLk1Dd1U(Q&6Fm@V*!Ril9=$0qPC<1)Z0dla}lm*X`)+2VuIftck{!?P>qv5PhL z;a+j z0iQAB@;WB6?*VN2X2bp_o`&S>HSBq!g-nZBOVSNIF=5wBX!l&qZ)}d@v}Bgki_aML zlkOVmcXtfDn#f|os(!3p3M1H?@{QCk7iA5qhU1D)%b;YC2CA*lVoKVIaM?0}ee$6= zk=OqWW^ut_eZdK9hE1nRb%R-r3Q2IhRteGv@6o{5p&%+a|0ZFaJ1_j*lwEhBiAp!h z!SnjdSo$eO@I1d2;?FMNU8{mn(x(9&e7BI|=Ry41CrdDAt_B-AY6vw?n~$2O){`x< zYf;yricV6`r|IQpO!2iu3{umAlKx{!$D(zZ)Z;a5nyH8z*f2<*7llsuSJ8QN z7myn}G%?}BYS`_bLE?7JmgH+~6S|W%`TSNQsBb82q$c1dk?ICzR?r)20 zK2_q$(yb7+VGs5WlE7#SWB#Mh1}5P3Vcbz^$0Y6Vi_Q5vkgf3s+4dUJ;%-~Nf3z37 zI5iGi9XVpvvjp=l93gYH&M;eQmqF_CW>RWeK$Gk3iE2n7ZoYG!5gO)0RIMU0Cg>Zf znOeuq^N-`6HLj(H;xZs);y_{><$=?)!Z|PapT} zK91*KJ`cPbZ}&OfUEd%5`2(bX`KV}j{a=4`bdUhaU*~@u0r4}PKYxX^TMyWE6Yx*M z?s$Qp&-s@TD){~UYGI*2KEGT?Fh=+JJYAnr6zYb5OYc88r~Ib|e}BH`PYt>c`uw|j zDg0f@kAgpoFYM-5L09je^Zy6&g#$ZL#)@>I{vG_g&sX|0eHpN#liF@M`D@yLUH|vz zz|Z=!TaO&?`mS>~2mW);#rco(xsP{AO}9M${C(+eI)1^w<-pJ9@;2Q4+xnk#-Ooi! zqJFOV+kCe^*w_65gTJl*Zp_8{2lze?;B3ZRu~#) z4PlcMF{o@i+)>M?SFc+__Wq-I#{Cv8==q3?xc`(cADV+YXUsvu&>b8qR^kI8Dd?4$ zMVj3n5@EZgs6BHjzDj;b4>B-79 z+rstcI4C)qM!)S*z^En-9KXPgylD{SV^<4TW7h>VUTdd*Z`F z_1Mj6qv8D#8K{YBfJfz-FtVo#(G}W&W&9ck`ERL^Dxw9l$qw{!ge`go*+HFC5hdGG zxnricsaBjSDx28DY9C3M^Sq7Q?j|_H*keBwwz!hp*K_gFgC_1{ssZ)zdJ0O8x2SnU zDTX+&M2DA>5E+w5H4=8h3(o>no)}9aw zDZ*Tv8IXDZ8wr)kVB&PVK+s~l`A6HE2nPL_MCi1hyP9&rl?@DjtZy` z8$)GWdV+U~9rfO`0`=CeC*|+sp{VdR4V+SguaDP2ox_K^kmCb5trv+*$u?s+-X;Y5 z1NPGOrMqzNz(X)rX(}7tcNNY$u^sjMiSjXS6(Bl$BV<+%<+E>jAbE2FZ@$(68-+ad zj<)5$9WJ9KQ%^zMUUNL0o`eU?l0n$~K8h}ChiR%gw5=qK`BE{G?k%q+UxqItDiTxR zH{%X6`l2;kVKf9bs`e+!lN!iKI|Fugusv#eKVc->21Avm4a&$&fU}*o% zaxNY{PssB-jf~jPGnT~M-UJduC(@X<2#~kdK|cp8*6f}?+!#6-$LQM-SrUNdv)4e& z@g#`neaINeL%6{A8qToKht^oBdM5lFY>It_PEo6%;D1qc-ho)XZx}Z+N=R0c%tA6+ zGR|`!$tYf2Ch`JLbY-aowWd!F~X zulu?_mzp#5tT;|Awd%=3yR$?Ss+d#NYkBEQzi-?5(6H*byT~ z{afxred{n~#2Mak`c*n%S}97Utp(qK61coxjLMtbrqvx{P^N2vIitQ3y~Ap3*5ad3 zvfYoLV^+WzC|>29x;p6iusvi%;#K@EVT?2GMuGPBaC*t!S;&LxVDHQZ%-R-0FG>D} zxk^@KVy8cj_-!ieMmAwuu^((}x{G^;^j_UPPrE;vpNg6tcU50 z^wY5Bp&LYi7P-FY9_+Ywm8cz3W|Pi)(RjGq0E+!PM88o zI(=lf>mY3Be$zjCMey!Q5fqF6CE~Rm@N;86bTwGAzG4kDbeb*ssx8W@Y*_&oq7zyE z!a1;X*bhqA3#?^h=CHN3^FZgQ4I8u~gv?)+k280zh5Gu3xY|P!CX*giA4~?H_2XGs zvyAPvPl7p3itIqdVd$Rbimwzy$<2vL^zEF>aJX?My|TUvR{vB)NyDq?9#F=tYHfkz z)3fR94P`X;b|N&@_yOwr`7x(Tgyc(sG5&Jlx3izOIHn zW;?)0aS<#uiUJX_V;DW^3YYxgAD-FYLl?U5q9^xh5$(5kapmP%tf!|g6lm5Hv8O4J zuu=qba!tWdtP_l##^CYx59G`YarjlGgq2Grk=?uiC*0hN>x1f{I>CwfUQ+?dnR)b+ zZYar&Isu-AV$gH{AQ`8YL_5tIA+goRdjEx5>wokr8M|5zqkb;u+$ztJk8j>$3w=#C zFZX5&s;2Q;U0L*!{zQ!oGY4-aD#6<3+ka_qq(5vNDSE z_%R)BJURs1Mz&MiJyKX;w4849Siy8H9b+|jwmy(F6|7T#$>f|#WTr`~&>y>GU~yeM zQ}Aply=U>>nmdt0>wm8xXS9{+R^_DtpD&T*(TdP>FrAUGk%V8Dk5a{JO(guE9G&!U z0S-88lD&c>F{u49_uf~QYg&^^mglb_Q!1lLh0{1%+Y&}(YYeI9=Sg@vu9PGl8_9cR zO@nghIM`V+9a1bjiN@3?^r)vU4jVjX?0n+L8$&-3qMhJwtc7(a{UFmr@R#Q{K#`0M zm>jTx4k7J7K`j-$3d6~rw;LfqFAO&ADMOsAhJDfgNCmIXV#Q(jG&>pxcY0xzK{jpu zmqZV!ghFt}L6WN?S0fWPlWkbz!ljlkr<1fR;25yDd}lPe*3JTXf#q*@L`CqFA(m@p z6Q5&-kY88?H#dl~lLPCa&$$yvaB5II$`XuP`Y@lH%+KHNg5zz>LGzKo@4mc=_l@@< z8g&(LWWsKYsC+|m1Xf4uq8?hdZ5gv=zZVxbD;vF%MOf2qxzN*1@P?BTOv#9br{g_H z>*&*9uW!rmbWNb%{73pS#}hI#_1N8iZotaAdbnwN01_5z)Ai48gO`;Ue^Krn@iNwj zht1vOXvP(C{oFpNKcfsibTd(z+sDIyy4YftWXKjG{g$IxvmlgI{ff0EN1f-n6=LH?CH;a?iTHSI`R zdukN?IMhy$u988A-^=Nui-(AI$w(a87foWW)L91ztgi1UjZ;=JxO#gQwbz+Tjm{~e z)AB-6Q|-Jb~RVz=qioRf5ufdYN2&<;!I7~`+LZ`J>;sAH;^4>9hd3FN`Htz6fr zyL5281huYv$;g-#VVb%;zdk7vv}hb@`8XaADQSVs(|Zu2f1IZ5KME_uKf)b1F@BH7 z3CN#3l1=<+$M<#4sig+|#4P6F%D3PoVZpcL$^$=7gnt<81m-;* zFm7oIDDDxw#Wq3IXZQ-8T|JWid;S-L!b2f)ZVF@#-=tSP-{F2`OID@5ide{ArisU+ zNrK%7_CvW0e%~?~7cP2_HZxRMRlQg;%OVg1R~)0iZ(E{KzZg6)m*DB#9Q>O6ow_U; z&E8$tP8T22;&kGsv0Jw{(6zh95~(l0==6WTm>*yWCHJ+lM(Q6eS6b^^Y#IPPUzCWV}W=I?I*_pF=@7aSiNqY_bR(v1J>z>fiJ!kN@#va~Q*gw}N zh_b79uBI*vQ?Ts(68@j{bE4oKfu@UJpli@C)HxCalXre)W{TRu-<_TG%bUsc;^+$` z;lgBmGENuwX6^uurNYip=`}6O6Y}hW{cUh$3LIi0sB?-QTO2H8Pc&bXF=K;ahpPx( zd?c8N1tjAAwI1A>D>JY*O9BQp8|kH^BjDECdvx{h19Xl~JhgjbhS5zQXu~5-_Gn)j zmHQV*6r0rHl7ljlpX3D(cySu6wGwK2Rk=R(bJSqoQj+;l0>>`BLA=sr@u{~6vqC|Z zaLF9uOk%jZ*OFj($$?f=x-MG55|$=%0CxgH3W! z=n;u4l&h$-pRCY(AkCK-^rMGgFIVgo2@70qpp;PyO%_b`C4YNxpgs(5^}oTr^Htal z-tNo_$pp*}5*Q=jaxiAS5B^F@<&W;zj8n>*$)Z3F?%MkxG>Q%$zhD5NdmJV!vK_mMQER>I&UX<>a{M%!LBi9vGTjXJpFZ?e!J|> zeQH$z72-=Tq(wr(m0+5EbT#iaWdYHd;g6oFog_&y1!f3;cXGBbER{-z%iE-2?=LYd zUZjXNQ+lw&$cXkz-XZ3{%5cvzdwxRQPr)y+oXiTCMXj#suzR#+Kz#H>T#0LlD@o<1 zA1;Ew$^+27=`o1s#*hOi-wFL1{Zw1^6_N3ZgXZFV+Aw=NrF*l;6}hJr8y}DrA(`;D zbsGdr8N>StRrI*41KIhl5x1UZaZGJJ9#0~{^)|g0?YCH~- zE&lB7`NeoGDI0bMX;3w1MV4z`#OAc9z~I&d?)vc-h!H(Q?psa*Y3W_ilo>^X78b+D z%*VKV$0(Fh+{_8sRyKN&gaSDqojJ~>2EnG zTVci4=`=%2#tL#v-ww9Y<5piK^SR(ZGuZUvr$owX0=pSR_;b50!6M5QrUungkE?Zn{8`IUIveIlM)T?Li_$H;H)DOH#tPY;f2g4g@J;KJt?GJ4E&BwIW&q4OUU zd7j2(!;cVYzm8ogF-Gw8TxFd9mXkBH#`Eu*KTwYVe+%;$Y1A&mjD7P{K;5);+{Vu>~=n~X5_RW zm@@Sk>Z~>4LPEyE>W{w}?f$7K?h#3wwsl?=Oge;T?ENXtHuz~W!B$6Tq8X`A1Fw8 za5@{>XyDAH#A3m8ICm`)4ay~H>6;l?K0bu&fAEQD+}=s2WK0321&gS+f&wSu^O81~ zW`Su_FL0{jFiP|jB;E_Ao@?VE!K4KS?Iifhu3_|D>k2m3GBAAdH_00iJb4+>MC|c? zSUzq)I7Uncuf0}mwNg2m=c@sO#Z#d2x+mG7+yRBZ7sJ1=AvpGiFQ|+w1Fv<7P&+Ka z91M*_V}WOCSbPoo4rhYhuKh6ko(tuceuZ%}SK`6Ib2#~OFTR^B&JXEllj`MHnW$WO zTD7^G%P}#<>t`--DkZY0P>Fa=ez+=E^|ZC-^YJ)WCw%%wn!P_O5?8k#z`|!*T;K9N*b(^*FSK1IS3il-t{@fC`|Bu;di@Kt zCN4vzRZoeS^b2yeU=-hFX^Eki1do&cGdjV!fEE@^vZ_L23@hX4wwugZR2<^`dRdZWa*8PLZ~?<_maxQC z8UjPYxF?apm^Je_%BaPG!-*&KR(b;+SryDIu)af-HrO zC_X?P$0dW|({il(`WoHltApNIc{19!mga8$P0aHY$*GJ8D(-R@gLmH}OT`0W%snZ- zKdTvItAn`aa5^ znct4KFlZ12T;qPMIzN&(HF(7h7*wNf|1ElCsy51Ye+2z<9mqWqPsMI}f`q*+yL(GL zRoQ)yzGULCvu-S#w~xVO_cl@*wu}3+Di)sBufvW9Z}CowF}z#sXFWx9DLe~YN$YO4 z09@N7WEpqi`Ewe$^js(?9}>ruxffymJ5}s*yohc4eDSMvENNR*&Y@L1`TKA>PMF*c z_uuTm>w8^-x}+Mr9dM^qygcBh*fjksF1xRkpI` zMDl6I!aIs;4JAS0U@6J`7fyXl|08yDzfisN;sT@Hj!0s zBBmDwK|9Q#_)-_MJhYUsdY+)RK9;;XeSjwIP3E<~i;$J|g+RzdEW8&+C+hdYm0S0y zXJWhcqtK7kr|%rC^-F^an?jN%eF%bYHZt}X!^y{e+04b5PT~DbhmOz+a^l@c9Q*q_ zqoH38!;fyzB{^s4-3vCf-)R|i?@YxO-4OE6Uyc;SPeLp4EV4dP17%n8L|J|=l>QBY zT>+xeT~{YjG&t0HDK^rgf-D? zBvO{EFv2E>P=~ox_tJ9EXh^lri@64>sl^yF-w$n4zfrII4b*6*HTvB)gN0WIAVV<| z_By$7ovFI`P|&NbFI^?KN^jGr8=sNF>xImTTdfv1y;m{4pERLFTCHYE!$~sXQUJQ# zdPoNYN5lM~TF`e%CGQ$dQA1xAj$QSpx$k>P%~&buh}cWqXUv4Pi`&qL?ILHiu5n8o zHjtX4K(aKUlsi=Pp1On`1KrgNu`0%ks2!h3-FlHJ17AAavk#iX8=zoS4BaZ51XtoW zVC=X+up3&5tur^^>SMzFz;Xn8G9i}!Za&6Ptq}Mb+zW^E#ZhEmBbjpQAIbHQ0=u+U z#xX+*7YB;tUJpywVN5KXGsWF(Z$J(Fy2AV5Ju}|M;;&|nk zWP;owNTEu2`?EN1dZBPVeo&Q{{cOV*S+&9x(Mk^9=W+S1>BMwWGxRnj5Myp8umS%Z&FjltN@Kw|3WLzJ=~WIWCzIfDYXcZoKbmHa|4 zV~#0Liz0IphG>TTGyBnARZrv>1vnZ850Fp3m(GeO(hVy<0N^Re*vB@QRE%ZUI5?h zI{HQwzvmwW!;;VX~pqo3zE~p~%m<@VAeJ#W@mWX;TaNSu8=~ z)bi*kr6f`&^tfI55hy74_vqbGYk^&$Njt2@!MIfo^r7h!ZtI_0blPuIdhBa0qfC|( zb-Byr)VBT5cBq%08ZE~<2wAW0zbCLmMqJ36jpxr6Ph-0mTq8lnY5c~>Yp8kHmj9`f zK-K@#Lmk^D`sDjMNO`3xF!QFfrn#f>P>D2r8L(nU4rIbFt(|<)+L8R)Eg^XJ??lu- z9#yjiI*4s}9)y`chQz%qpyl{c{?m3_HcQn2u55fp*_<R_%7qXlOGV&=doIz#UZO z^NHwuD@^Njg?N{7{720S$a1ivb010ZPlTCQqo4qmtTH1?vim{(v>5ySv?Tt%-~@%? zJ#@ECD7_mYLFd>Jy624w*&!Au%rc#LU3&veJ-mg;bfz#PR?cG{7r8M1rTt)HT1DB- zpFeO-(wg|dv;(EDvi4pA54pO zl#}ltS5lJ&+wpM8Zk%mqhXtc^>EiGN*ez&RZCBICgn$gl*y&5Aj*W(!kvfo;^$1FG zQ>_Bt-l2CF`LT;s707}MH|XC{J*4E|UZz0s+8!^M01fN6D&lHVUEe=Nq) z!|qdpE<{4;EaUp5x(cx`>!=6aBvaNGe_extFf;}ACK#BmZ`IuJE! zJt}i!@tEpM^46~mLffKHr{)={cT9p+pY0%k6J_s9r9-@2EioJLf)-ujIpQRcxpoH< zoV`K(tfAq>bvXn7`^fRnzr@v_T$>&$zRD4n@&d+g_8@jlZDt<^>vN z-GF0H3V#3p!ss2(rJT!u{lr1dmQ=bOBJ1UnV9^#Utbgo)PcFYAQ`U_Di~AF5(%o2; ztK7z|WG@3#RLDIvju&zR9$@Bw8ylD2r|j%>?o;J>IP*D?cqRB@`073g{$&lxlg>e3 zND7diH#H5eE;Z+;_7gwjztlTX2IY!Ui1@oSxS05zOI|RSJ175u7VHy&2%BRhVx<#! ztei?dlx`*(g}tzD!%cV@=nfu}7#M4&1vcGnB-SpPxY_!_+Yy0ae$pAPmTHn6ou^Py zd_3Tch4kVufe3{<=qaut+~osIQlqml&&HB5FGZ1GWGfw5hmsd3F3G zAt!h90!g`{O6!I1+0l46)GR$gVjs>xi;tE#udSS@_}(Q4^GvAF6K9Um&}nu40G*EHp#M^Oq_=Bxx-cA<=d}1s`UL%Pbpv^H ze-6nO*a6zhZb5C49DCFz92?4?VO#QIloRcs#mXZD2D~phwksafA1RZhKdlhbIYQ`1 z`H%74-VM8j-+^5tchGUKHNi1$3r#(=4=$OmgN566k@(gA=)YM5#>kz5vCm(So@bk3 z7Ty;)%)wA|!w1uM+!1;yC1L#G_1wX#e)@b$AVySmF}ZoCVC$|l()cup9s4p4w?AD; za^5Oq=ej3&>178u+UgXp+fqfAoN;Dtzn($0Y-xU0zXk5zbDa)O*~ZIEf60vb5{>Jp zRwEg)map?SWwlOwV@yLSo%LN0+Tyo@VY39kA$BW{Un-6IA_W5Xq!r?~b`rk04%Se8 ze*X(Gl-VrFE8mFWYnv7Mh2~!|M(Yvo`(O;-CR;G8Sd(pyIf>Htx@^iMJ+w=o4BsBt zL$_rP(d>|@hA)JDj!gC5EZR6H0++a-rREcZ;A~73Tr_E* zAG~Iv{*)N#dU*%m1dfM+b<_E8TMSV1gA!RcC7FIaeg%8RPUZGxNwbr0h6|p8DyZ1y z!n+xCL$r4vOc}e43Wpi`1iv5)4?A+VvU1?V2_dI<>LGQ1T>yD*J|rZ42k0@j)~gf~ zh|^UmxNuO7MvM2;ISxW|a9KW`BtM4x@qZT0jUcM=&K83uBk{h02}T!&P~$!?V9dIK ztt^62UYaYM7=xW$ z-VAAL_1sZZb!Rd0fp7G~xC^8pO9m?VNA&&$mFo5@rPM=U6>n)NCX+Dt1fsU)t zlWoSE#m&Wz_H#_CNl|c zp$;}ajfJV=qgdDfzTt!(573T~fnM)Nu&Vw8ecjbhCkna0y_@&oC;fl8`rZ>bG3^Mp zhNa@T0U=XweUqIQpbGg?e;AQviEurAJe$2l$g=&(0{@4P!CvDg?H8EnGW~M=ppiFQ zxUre&HuKQ)Wh9^DY{fs*GzFNMjuxHGWR8L{0J!mcS8H$&F7Cre?e}n%WhZWw=Fr{A z4c-r)rPE}7k&ZWeA*y>S3^}Di;71DmFCt)fx-4&VXd#j2DOF5~KzCPPD$|}#wcfa5 z$k+vZ?)?YgKSzpKZ{110C_AFj&|S=#o(>O^MzgqI2V4DOfGz!p3Yo0}b7?nfzuw1I zY^o$qYi5%r`ejgkbvH~{s|ecP*W$6#_as&CG8hM%Qk_O&e>(FOaaoi{H$+`V`TvHQ z?$#)DK5&kHp2fIyO*_cSrm>{A_6tpS4gs~TVo<4R%5~`<1uyr-IKrVB z)DtZE__s}9q_-8M736sx&o1cvl0g^lcjcRI2zS(AUp{)Oz?eNXD9j!+f&Y>US4QPR z`i)+A<&;7D$Cw=9r#wBYjmpOY6$qR}_Pp8t4wi1(@?b z+%tiqU~3DV_6>B#bRj1<=?CL!pHAL(xX}QME7qCh1lizRN3TxL;7)r^1l6zaXs_eU z8mCWV$U}iu?WC;+GDqif7Nx66`|;(h#pgS$eX|&*o2O$}Nf2%h(1*u55#T*X7ghy| zpwTWPv=3j*X4YTDfZSI^@%L(W&1fGsUgs!XuRojC{ou3}auwS)O2b*rHd3%R z2StOz*=K}aA2PnZkc!=->k`kownVaX32caq%*17gko4)l?k6@QbD!#I84os zL#uE@NZBq*%sYGO#CBJd8}T2ypI5{8|Hg2?E0cJYhFX#&*$g@X@_2gk8)p0VXmFKD z#qKdHnf)PU=oQe2k2f6?bVhT~xONz)d%I%HgJ#mKXbRu-g+7aaZ?LB~ml){IMS~;u zkGxnV-G~zH}+$>!|tECafLy9e<6U^aLhPD!?=69g)o|1&bGM#JxZdjL+R7 zSLTUvi+|1s@04nKW9ED7$kt4<+N>3_)Oxu7{JqF0O+f$N64Ld=lbqijhPTJs3ci-t zR?#;9s>&u(fp5KAbaL%O#!khT zu^;^ySM`Mw@AfZrPTyACx#c$Q=#eBcZ?iC}=q~2xKBHGWl5uye0$dWA3*{5GP|uw| zao*5dTCL!OqL(jV&4V!B|J4@o6WAAqTm-Z5k2@Lu{)Ju{=nRlR)u4L%FAOBzTs1B|2_bfB=JY050t)>D39L!8K;gJDq`= z&E9%sNp7O`z@$(z-EukZSbhti+h$;}fiix08xQ-OMS0FI6Tm(juMQ?-k;-*^+V@Gw z$$v$U=4J5CEeVG=Mv$X@Bl%iE$9FDwggw(%b8^39p)@Lu&Y86y&kyxe*&h$+>Eb5j zc9{@u`3^L#n?t^|&E$_wl0s?U7gT%nJG6-n!NPZ?IPuW{y`JvMr)I122c$>Ao@Fw8 zyWu1Z7jLHx2ivK_)fgPFdY-f{+Yfrncua6qWE#4IA^Y%Yut+Y#^G-5+^?hUZ^Jh{1 ztF8h0aiD{gTYtpLry_I-R|HJtHn?jj&%Q`c=X@p$&(h#yI$Jo0UYBkF?N$45*Uus9 zNmihVXfYNCcOEa671NP@2|^CN z3iVc~gK~%$^e*d!#bLnvg&yPn==d?(&CR%V`bzpwsS7gMGfc$N{V%ycM3;Q6K2n~Gc^xC zkE7lWqo`)vHxOBViS9J3B0Gy_2#la$Cbak@ZS@u-JwbPvhbHxm^MhizZ(jhH|EZ(X z%j?#km;a*Ma+dI)V$RZ7!NbIQw$=CvUYz22&Nx}GpW60K!}u}bbc$pK*&S=Z_?-Vi zh8t4pt1$w%?5rYMY$p^or7=CKi`e!*B2P^SUQ;e3TD{G9FHHE%$Mo}OwbDt*Qfc;J z?mU=Wew59Ko5jz*RzeoJ=h8opd->gWb;n$;sB05&{4oXy9T?Vysr>kqanwgP7TrWIa2_Qd z&=``+XIEcD<@$UGJ!;P19MKQlf#)C}rpc<0T7*Xwt6=l7PayiFhdxx_kMmY(pv#Aa zto!2xJh&YAK+Ab#bc`uub}S5)!|&0X{UwwH9L9?yDq&X5DE4+=7LnXM0!)(XIX17B zk+-!%!_1xZ`J~Hq#DEXhR(&R2%Swq;#u8X$%F6}@ zKOF{*^Qm;xT|Mv>=HcSd`WlJ%;~+Wn3%#>ygrtO>hZ7W%o%dbuaBN67k-M#b}Yw< zq4R0|%^@bKVi77Ps8H5p5BX-AgF#--MAvI3zF1R24!C*3MvqEJv(dw}m4ZNs`r{cncHPzo@uF@fo0EmAs#W^2vz|ndcSg~LPiEa>Bw^pX`++Yiu>tq8eT%;Gi z?84?RzsT0cbRl2x88n)g;M4c%m_F76A6;@HK|=t~Za(4W*{Bh*uxJ?ECJCwca`1M( zKE&v6A$u1thfush_Ov}AOII%-*n3G7NJGF>m(M86MB!QQ=G)%pJ3=UZf zXO~w9o%2-(iU%Sn`*9|^TE#LE?;5y}&^l@n`kKzM&|%x+5uS=afmh?)Fm>b^P>TOb zTFhEV>U?wTJkrWNcS*J^x`f_Gy z`bLb~%u%| zZlTtT2F}%J7PLN0!s4HwXjODLraF6Ju<=Q%e_|%cG;QV--u)osBRA4if**}sjpNs5x-y5m%(YGlpr z5^*wjUK3*S85)z60pT$@cunac@to;QxA$bixY5;Q!MJMj;DH6q-*pNr*)a0NYa+B2 z%o0#?o2aJ7LW~>RKvaem(e%n-)tp1{#c~mPR!5_8tT(p* z8jW%l8_@g~FOwT;QX;BjFj+x!WF1VZ|o)>Znzw>ZoE`@7Feu6&e$luTH z2D1cHdhL}8KfNLb-#299i7y8+_Q?+`|A{JiE=HMhcA->m;$8auusV%vjwH|8>rqO! zAL^{#Fs~jk>DglxZ4$B$k6WO>;}6|EX%R&Fzoeo@{jhy{Jq{>LrQzu-uu_;Wwr%@G z3#b2vc7H=!l~_U}I+wD)4c@^nW(7XGybteqF*w)v4?Hi|pb=?${A77AqL3X6F+-jp z-E*3b)q75Mu8ts1J?|hgWhYh3tHztRN?^{y9pB)_};WV$d%N!lBWvbaFv6o!{|*gr)Ds)uZO4 z%ENclW`ry_iwRWplz$+p z|F?-2|0<$Z^$&>6ksIW(pfP+iizGIT1=sW~nQGakQQttGoSe>bpFEF|blVV`Fn1jj z^<+0XJ=%|oN6vF`Pb$gZPYmQa*y3Nu0Q%n5oL>20kKNl=Li8sOxb}MscK_68RYr&6 z`zJ@?-EB!GFuD_DG8{l-ETF5Q7WH=W!CB-awkXVCZ!Sm#qhCBudU+1Q$IW0J?yBJ% z$Jm-%-j`_T@v-#a>MZv!Jfp6IXGj~9kEqMX59 zDn9!%`S?+XFSiNi;tWsG-HU|o(4&Pk^`r$`A@F3q#y{aU24906{wD0qxdJEbc>_H$ zaWc**jR%p&z4&Z6Xi*- zqR<#4{)Jy=eQ<-Ne=PElJ*;LQRU*KyAZY`qSd6+z|)CoFQ3|w~v~izDW0u8;^He=90t83FON}GnCshh>N5r&=*>E zcx(s-w%JuM(tm;yIo?>k#+!iCJ!nON*>>U?67LE(SNS&lo1B2pKE0;FXNAwG=o$Eo(ZwfM z9+O4Bg(SDL83${HJEo9N5_fq(T9qfF&p;2jI9Rcp-%p_d&!cel)L1;Qb{(?MTv>0g zlk~9MC#rL)38cP_VXfA&?AqWB+{kkh@Hj<{m3OoUw~iF@DSHgq%Fe-6=994b!hD$4 zdl{alRe{qs7DQKk#rEnUTwY;Eef(Z?(z}Dve8W`qd1(zPiXLpy`TY=PJP5<21@c>J zU_-?uEP5;hRSHk>=kp}m^wtfT6Pj%PiL0=1Xc$snj3J5^CG>9AC$gum1e|6|V7a9P z5%+S3+T%ANM^6loem+HJ9h}M@=u;xLW74UMvmtFBXH41#9LT~jO~8zP*fiD}!v8vg zk_>7c>?{d*m0EqzwgSDgb2zy^ar8I)26_8saK*T7 z^j6qznlQl?9sUInk3%|wPA>G#3q4m#Mbc!nx(vTLH;tZ8@uE%*JBihRJ~YWPu=nkG zEDA`qzE__BevCP+nK~X$zK;h89a%jxx{_sN;n#ivQZk75mSy zaiRkLnKw*#MQ))db>G7c*>`Xusf@@B7_hnu<#^jmjGg`UI_|jlnl~o(cu2T%<^2pn zv#OIQc{&MiZdK&Fh5KLrrMLKO{aXIpg%A*lUrA$1R$*QGImjHb9gA(0$Rmjb{E>T( z{PNcm;3Lms_|bIyH2jk;(msyE%~xq(_Chjt><#R)&%*-OSfOV_m%V7)!v`GXsDG3< znWBBP=3;jpiiAn?sr@gZ?41n%O34=!ic1i5W+C}_u4eCtZ+Ils7yPBq(7IWL;Fw6z z)h_`3_O#*06Y{8~d!KTL&7e@o0A+oPMzr0+j#_aL*z8NzK||pv`mFTz@(c{p2!#Al8fzMbDe>#f$3 zq!BNOpW#%DPHv}F^b*}aeF42tc$MrKFsV_c%IM`K1*TyeF}gbjZ2oSAu6xCTuOYl< zy2ltW@;Czvb#(;Z=q)(tR>}7{o+ASrd|9(St$3(Tlb!YYG#HCMM|lNV%nAB}57Qd? zk9SU!+h)Ev)IA$t1RArU(uwFl91dxdSMURiukl)SF?7?;+xY#(7A%FsjKg6$eu|VO z`$JrZzg?uppBnV!yZ0W(qlQT&;`J={alt}kalPFRj= z*CWWUZN{uo?spip5k;r`LBad850l$X@R_PPH4<`zsRb5P<3KX@Oh~7hOO^51ov|1j z(8MVB8M8`91aHd(9W?wpgWW3T!+&#f!Ab9zlRI5v_@^6CrdXCXST3fbx8~qK$t1?8 z@ikt5C4;8c%6Ms(j-a!h#eHuo;P3Y}Sf70r%`Pj_6K4j|)F2<`&&p$<}{HiFpK9~eLn|D)lKO16| z6N{6vjE)0!-tv1Yik;HLL(!jcg`lZaY|Fs%NL6-Oh8ia5<4pzH`~ zRGhh+<}ZlV!2Dtd%5XRidtC8R_|8Z*}_2dW`-# z9?uH<{C$1js0^bHtG|@e>}QtXDpksTnBgF2Se79JWh2|DE)q*f|=}T z4%7EsrXTm`GavVf@E6P*(8KB{-LOQFK8FIdC>q9TPfX~YuhDqiwVugOL`czYrJKFX zcsa2iGO#)T*KOa&NAI5uZ*NpU zi_tnc3%(TV*3>HggY6qasA1_{usxxU26l6Jhr8j}^mr}W7yLq(DfPHFe>uKVx&r@Q zt`mG`Dp>K-n%_A_9j6uCB@bD1a8$fTcIJH}i=-al*=yS5YU_E_%uT0l50=4Je@}eX zQ)pf876HLiqp;F!I;8nWlQqjX!qOHo!3#NoTVrHLw->5V9j1pfnwUmPUO3Xq`!(Eb zeuz$x@@5X5xyXAwpf(1xxmgPv;j`pOxLGy<{{+_7d<#h=BkF6>p6#O( z7W6|+h7V`BU?C}SKZ=)RK2XO+--)~beY&eLgloOwOLEP8Y1vGZYuths>jCS-|#081&y_dYJI#7-}9?+*%XF- zKRONb4Q9fNyT{SrxDqQ4{iUAM4WM)9ao+0V30!J2L~D+elJO^0+0ieaqRNs@Aho=m zw0KU2vs@W53R{V}lQ+SJCFcCTN0zv9?J-DxJ6;~W$T?DZ~16=)Pyzzy35m{?6& zejD2eH@_LPF|W&ELQM{hiMhrpPJBi4G#!wuQAPTy4^B$lg9(Kqu=3A$vTw}{A~$X$ z_o^q2>tCV7Rc6Uy{P|T}t91!?Zv@9g$TgDraqhT7bvbBuHqzCK8aS+)Lil)nDpF@g z&&*GNZgUBEUtmfebt&LN|7>!0e$M~eJM(C)-nQ>k6d@rLGNdvzDJrqgW1EvCQ-ewx zRfZ;M&OB2{386tD8Vr@`I**-5hNMZF6q+u%Mvq6*1xSs~; z)BDlbx|xd4--&l>KN718L%764`H(qiH^}I!ve_e78Xw($0d>ymu^oKg|JG<{QY4v( z9?GNGSltMKNgaKM!L*v-ots^l`j}M`3?G9ummb{lkwel4615^QDOWtc9~uS zQSWf3!F_8{?bAoR^e7P5>SSV5;BAyPatFVRdT>~h$i#kIidU~FGPzs%d_(g`nzd{T z_Ep!Fc_W{jjB@4X4HqFHqc)Op-@>?7wqv|1F-1%y$R5%NU!W(u)MvF3hyh;ovUj2Vq7B+^9@YBtDm zl^1Ut1!aCf)n&1eT{9THS`YBL0|&MNhH--`CvuJk!rb}S?es3LrI$+bs8QYublGNw zi5B9V!iE(n{cJf&PgJ9B>nh01{zZ6cTsGNfJsM2Y_?{L!z7IvXsZ3%eZ>&z_(AGE5 zcwLt?Gx$pt{i6ATWY{%RliLgp=~sf55m|8Hi>=^6nmkzF*)OPF^oD59=q0+lw}Mq; zB9S*C^yl}XWZcDKn928EbJjuBZqyFAv3n=Fq?OYzThh_M+ZcNi>tRu<6%BB6rp(iG z5E{4=&nJ&TL2@E>pR%7=E|4->{wklZYabP~+Ew7XhD;J1)JWg=*C1EVd%w6XhJ`nV zaym~R^E|j{(x7ca-PhTooZuGBnP-OIcN}5+Ly9rbLl#0-#M5lw%CaXN7tu|JXK)86 zU{{-F2_w-A=iwD{Eu%1}Uh_4CqbV39r7 z&7|TB*WE;QQZyMoVJ^1+*hN!%g*oqprzoAk`vg0lgL7JeG~*f~ZEb=8s{7L< z!|A8X>(#-$-ED$7&b1I#JchxM7G7mOU>3eqp_2mI`a(ta|l;{K%P9? zr^sFVO!47XFB<2PgQokpqSO&3w#7mO6pyXLz@Zb-`GW!%H0~yITfzk|UG$)%XFC~d z6~4sBN@KCw&yw^V)+FLt(bP|nNyG;VV~(y4#tk*zp(WU06!g3Qpil&qx&Z zbAiM)EBXCQ3W~*5!GTZ-Zbr&bbojXltXl&>!fY-n>DCdfx#LFFWj8b2bahm%4unXL z+q_Ocob0GcC&kUD$=ALdW}tZ|@!>rR4sMu$JBn%<-|9rz=_QWaPU!P{og;+tqQpR0 z8e7kbGV4=?V6{&*xz+rce74OZ72m>0yv!GY;@5IUtxjG*+|S`b^K|&=F@;Q7s}4E| zgP>*I2n2#wB<+C_tgBn4V{VB5K z>Tx`5^BNQ6<*-M-94lQf;>St)Z05Xj*z0!;yOLkilRS^!R45a+v?_32LSwn_d_Oev zstVtV$#T~RUZRWRv$57N6Avg>Gd8m%xbtDkxFo8dwvJWdOf}M(>k6T`DCrGenl%-~ zTtDFR#dq<{8V5``Va^4aE3)HbKe4B^gTZcU5=PvUV~a&}u}jHBhAuAZa>?0kwu#7p}zlP^azJM7YGC=Q{26RnZ2{uW3RQ+cou~;I^ zY&xI>B~D>zoEi?ZCcc4iUP9t_a1DfdNVAO+)7T29VQk&15MnwsLXd-=?BH2KtdHCl z8n1N>7hcT+xOqmoI(k{zN;Cx0`CM)!khQ2OKlGo3w$p9zz<7@ zCBnHYQ!C-{-(!a`_cficyq6y_Cgpc>s^6cJEq`>L3zlFu&`$G0Je1e z8={Fav?fxA-MHQw#2x2>Oq?pZ_hdt}m@LRR^V+K$V(jdyL^w0PLXfs19cEM?z>tm! z2wvfkSvio6pZl05l^CMDktut1vI;mY$)r0&R?t+zTe7Uv4{EMwVMi+9w1$VYzSka8 zO2?t))Cbtwc9O{d(SD8pgIF=QHBv%Uj_ zevHKnp10us%~bYTrXtzJjbA&n{@wy-1i<(Pm~k5E0^GQAl~K)2cUX9` z4SyEp(jM(-aQtx{PO24yvqBYkA9|0AQZ~cZvDx7E+Kk(ywT$y=O~vM=b4b|PkAhtg z%>`;6CCy$}Vd?sK`nEE(G;lv9WwbY#N7WHVJd5eyoI}kNe?!OE!*}AVN)B z9)MN-0XQ?bl%6{=iTk{_jd*yx=kwBeG_o%d3dSv_xuc~xJ_^xO2sqR;&5Sb2elo7beh3!{Kk8biOp9f*CQ)&LUcQ=Q4WOf z9bv@cav03>4Z!y8k#yeDS@`I~H|&hQMNcFtVwI03P59!nXHgk>&7S3gU1UiD@ z`u!-)GkEUx88Fw9x00aH7#epynVhFQXQ9&@1GNkxwe*cZDKiR(gIKZ3in)Rz*JNn; zMMs(Av+umG{lsmWKe63)WUHS=KI(o3i< zl*~Bp>lI9#>_8$6^}vs3TRrA^CIxX?YzT9o-2NO#)Y^p5(vW9)4Nar#YtE1;r6p{9 zYyfOnB@BMIkKo>HWxif2#4Q-S6<2#^!I2xQ!DV$cifrQbWKOzlgXUvW2<4C;)dw|S zv#9j>hwyoL3$O8)cUcZ?FYqSEvsb(nqE~Xe-U7BdU*gHX@Q75lW6@zxC zam4gw5A)6;nvQE17%e>V2nsz`!ieSv#OdrqGHIm<<9c)|K07&$TyFCQ!&^Z(z1oNo zinhnuNuh8w@d|omEobXmQo&`v6?i}KAr<#G!SrLcV z#E;^;zOksRGLppaSq1(&{-BtULwlB;7u@#@#KQG~d=F$b8Jf|Ay#-3lQ*UY1p4~zp z=J?aO7LRD?s~lYKT}-xfRrHitJZ(4;NsR`z(aOyQf(sL_;(c8sTCwsCh?Fj3Z&(gu z4(z{!2JZT-`>0Ito%)>iwZDZ)(T89_el}?l8OB<^bHV!yL-;urfXi?bcKwPO@ahH6 ziY&YbOPa=FjJz0men5d*owy4U77;i+*#T}Bq(Z>ceSn>1@J_!Kk|a`y^qp^L<}ray zTs0q=o+P;GDTQM!`0vX;Z`OK)JA~aD534rFLTvK}I)v}-amifNb;maHn_Ay4_aCJ7c?|V#AGGcJrCp{RIBhKrB0?{aD9u|C@g&XrH)3uBak|UG2 z?(Gur-R&|_=&llsvE2iEe}=%MuY7E8paG{fWbxhBHag~(CcDb`9W5N2fIhdy(0|G! zV&8O&F>vZ7lNu&)uS*1FpXUjoL7NKn#n(aJI|UeX?jrcuB?)q$InknWXLzD04c1<= zB~m~71cm++!6k|JL-2 z<6)VPawEw@yP7e;AR&^HtiV=M`RLEcweD0lWzm+ z=k%$T_b__n#3L$xFiWuatv1v?cp`Yd_y&2XvJY#W9Ef9JMVY~tq*9GidqJN=2sJaX zrx8mQGF^M!u({%N8T;rsX<4-lAGbH+l8GfyWgbY@oeZI3>FemWTl-0*>M`uOs{qEY zkHPqg%NUTm5`L(sz-6iTNFF0TF6C>3pI?yNm)B??TaDwsoPw3L`s}5)AS}N&3@S6V zK-GH_XJAiY>a-p3RMJs!=eeiAVtxTky|9M^-AJNjyA^#ioM3Ue3>++s$1!JX;Csvw zlpT^JkjX89$No8}^W2_OFD4* zv-?6YC|iVc^Rwpr40tcVK3A+bvja^@H#h&=)9d{a{Uf2N8pE#L2(XGt&XzWJcqZy=k-*>Vrco|o8I+pQmo6apiKY$gw z8xK($;#foKA+V0`5tmbjTKlbZlz0P{Q!iB8Iu>)KpV2+SO^}r+hXzWC#+|DcV@RGU zNt+c$52}`6{C-JnFLcCD?xR_)a%tkShVXs^;@I|KGNe4qrQ7-=@Z#4oY?$COE{fbC zFf47srcDmm-E7WHIw7EWMaPK3Ohqta8;#&9uU)V{0GaGa5GPAP$FzeHb!?|QR>#rh zA*F)#M|)t@vHQ#&HXP-XTLi@9FpXcb3Ld(!kiSV3yJBSF%9(I>0+_|2sOUmsCC- z@6%1yx-Y=i`M1#9(+KuTHIX^H0tNTO!=Sy}l8THP55}XSkPC`92g2#q11^ub|C%1R(Sl=_)VaAMJh(mkRWW#m3QST^ z=Hf)|a3vO%q9`NJ8QTkEk(w!nYQ4hkM^0jpT?98yF^VIk1l8*k!Fpj2ZaiL$ zITWeqv`{ke_!2U1$TH6Qx&k^q4CHot2Vse25&d3vjNN@>V^#sL%3&bG6*QE|-Sufc}K+A-|or}gZ^XFV7;Eu88*zBf``J%epq zdldYitV3a^MrdkK0%tcNvUAoZ{FzpSZbD-~q{NMVWg|s@u{^Uw>D1+v=y#Vdp~Dp>V8>n)AL#w8+x-&;*cSC|aNBnk|plWC`=XpO6#6yiB zrC=gnac<;23nJjvduy!M8c(z~X5x_I)!6S82|t6L@T~Q*WsMr4WLLTibjS;#h243y zl9b_Q7AX+lLFTBgHH%rGUx$32aO>yHV^Swk+x;j-PH5-@{SRzSab`r%b`z$X76pwICN}%w|;TCvm3_ zi^8ShwcrxEkzEsP2VX4Cdj}j+djrp1ROW>AT`~As6b>2Yiv^p6 z$Q<`NDzoG*er?aiO11NtJGYEX4C%o4hx(a5UVr+*smORov^b~o@*WOV5u@{?f>1k3 zjyZVb5OZCY*KnIO(5(++x$8p;&~VWbrn++q#D}ZVkusiScyb+0YM8<0GzH=Fq1iYz zSe}czm(Lh9chhw$Jvi6&09l}rhtonXVz96;`nc=k>im6Zwnq|f)}F>Y7LjnqE{_^_ zrl7(pYdpw)L4%hzocqO*oO5S1y}~>oZ6pX__bD9vq!&H)9>B|14V;tBK==|da5d{A zm4{_HciU&Ubk95NX>h^f0j-3*o{sZY8iD(!Y*-fgltj+=#OY<>)Rah4&kYfEsg2@b|JH@!f}UyT3O~F55!-cJ|XT zTMWq9+y!{V=N+{xZzNg!n#0=Fq`#7o2&RGB7hG*RS2_}s>%vgO2;uO;^Gw-)47 zc43*4H0RK5g2gw4ID?>xT*IbLI2ojaUfOP0bfp#>t!t@a&0HK5c?|<-qCqq{0n5$OX-=URjplimEtYxkldm7-jtYRGZ&t$21@VIJ z71=PP?JnOdv5V{tNJOdovJl`mad;?R zofM4Y*GAEvsUOg~dp-KKe1m51D8}wd9;)i!ri(9(MW;=-1V`h>(;)vE+Tu_QP76k` zNA6DLdmMu?!mOB7Ni^Zn2tL=g{|2FwacG!RLdfO}+I6@FCvGo;w8DD4o%jydg*pnx zIlrcAY60}>wx!s+-**%*B1JIb`>+L{g4I)aIHA zK3(L6H}|gqo#B48>>)_|-A6sq(;D^`-FK>nJR_o(64Aez4$;6;q@fL{C3pY4Lao zDE;08qDxiDV(lzg|7tvVy%6OJY%1W|I%mkwvIlf%V7R#p*qvYf(e=ngo+ZxF*m+;^ zE$^%OO>`HSm2h<4kI`V%1SF$Iv^vOgvqOO*j z7#?F%9y@^Em;yZ3o``Stc_(Y$%idX24yOC8V^D?lQ$79*n-A?GKR8F=mZzRq;!i~9vEKWH?5r7@k_R$ckd+re@Ps((9yWGcD@-@?eJd`WCqdv>4=)b$dRV`4WYoc8`uwX6Fr7nb-Ayw%k){VLexYqU>pAWx8C1WRox_vP%bNXbg z`Ft+1UneGTTJxM-kcmbce+Q=IQC96>c}Ll(NxB$MN7y4FE?t|xq&nrAI0mmo@lK8NglKY6T6(RBr%c@ORb_Z zg8@6pv-?#fE-)OrI?6!EFAYqO$FTEf6tTkNjp&vi*RV1pliH4WPF%NYqs`C|_~vXU zFw~V2K&LHxp=A&^-OfN@?R1(Ay|CL6z&O6|4$t;j+{QzTI69rQSd}TVkMVb2> zRY|(1FiGC6NlPqkzhL9K*2d?qS zX!CeC+hfYd-*;X?aSz|~6}FxCV;hI_ee~!P1u15e#x=~J-A6Q{dx0~U1FFwI5!17_ zTy=U7Ov;=DQilS#UB_a$+af~T;sU-FZ=;K?vRML0QDagkc@(~@nvm{$(m10*9!oNF zkoXs3yx$ERp|%>&f11ebI~7DT__(JyX92l8L4v*Av=EH19W-7(><2Dfa-9x1H55l3 z9)KHp&(URr76Keq=XTEwf`qCEbm9R5b~7Dt?QH{E5hV+Qw5Nc%+)3Vh)P$*4dReBK zXhasNtt0k_c$TAb7CMYEh7Z@Hm`W8fu5amCx2y?jzW}3R zrO_wOny&EVne#CV7^Wpia605Meamb7cC3nkho9on+bRLYQrek1SAV|8O^Sx=JS5Y$ zT_HfjXynrlI@)L?yb9I8VP_1n@JJ?+a(*q4FJDAn{G5zeB1&k5jyl%0b`X&mW!e;5 zPRd3fCMzAMKvfu$=Q&ql@$P&s0@Uo3-PQcl`GZ|A?by`1#koe)DzyJ+u7lUH@Gx@Mn(JZ+_CS_P?Le|Bc^g z^7n`R)6af${BFMZ^(&pPevOhs>;56fPo6*Y_x1c6TK9YWKj--$6u-ZF@y}qse~riY zo1^rb`}ODN|N4E!n9?qB0Sy59tR z`E}f1_nTkq@Ymyi?+R$!Ckc!`Zvo{Tz8$c)ixw^U!K7UF!h6qeVDZ5V=wv`IkT9GqdGvDc#F9E{$a+Y|z^fcFffv_VMrlHnwp;YxFRJ z&7YFZCR*~&4go=I&76a56z(l)03^N@0~bSOm@NH8Ow= z3_QWE-yFk+H9RJMS$Bx7>|lI8_zKnP?4d7i>Jn<^!{iJ~Cv9P_#+4+JX>K(}=PnVH zO`HisChOv?jFG4xDGE!KJ8A!dM|8?8Iq;gCy|-gQ4Ek<_`s%E`(Y!E4P^qBWp9 z^amA-(8o<>JE`BeB0_J6V~nRPoilY8*>tLbJ`o)R$}@E_!LE)rtIq_T<`;a8o08S3 z8sIPe7|sa^Q&^e!FJYRnJwG?Igm4w z-$z!KKY)sMWA0_@NNP4#86FDDv(>Y9@p@)G!R`zh_Ce$$9MU_14KX!ivrk(yHiA!O zTPB3C+WW5J`Vo`Be(46fSUC@mPdv$f%T|G9?`CoTT z+W&Ef{aVKX|FO@g{`y}p!S8*9*z5oLxc_NC{YQDQ-=E9h=2.20.0 \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/seq2seq_lstm.ipynb b/components/outlier-detection/seq2seq-lstm/seq2seq_lstm.ipynb deleted file mode 100644 index 8d66c67abe..0000000000 --- a/components/outlier-detection/seq2seq-lstm/seq2seq_lstm.ipynb +++ /dev/null @@ -1,610 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sequence-to-sequence LSTM outlier detector deployment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap a keras seq2seq-LSTM python model for use as a prediction microservice in seldon-core and deploy on seldon-core running on Minikube or a Kubernetes cluster using GCP." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- [helm](https://github.com/helm/helm)\n", - "- [minikube](https://github.com/kubernetes/minikube)\n", - "- [s2i](https://github.com/openshift/source-to-image) >= 1.1.13\n", - "\n", - "Python packages:\n", - "- keras: pip install keras\n", - "- tensorflow: https://www.tensorflow.org/install/pip" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Task" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier detector needs to spot anomalies in electrocardiograms (ECG's). The dataset \"ECG5000\" contains 5000 ECG's, originally obtained from [Physionet](https://physionet.org/cgi-bin/atm/ATM) under the name \"BIDMC Congestive Heart Failure Database(chfdb)\", record \"chf07\". The data has been pre-processed in 2 steps: first each heartbeat is extracted, and then each beat is made equal length via interpolation. The data is labeled and contains 5 classes. The first class which contains almost 60% of the observations is seen as \"normal\" while the others are outliers. The seq2seq-LSTM algorithm is trained on some heartbeats from the first class and needs to flag the other classes as anomalies. The plot below shows an example ECG for each of the classes." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![ECGs](images/ecg.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train locally\n", - "\n", - "Train on some inlier ECG's. The data can be downloaded [here](http://www.timeseriesclassification.com/description.php?Dataset=ECG5000) and should be extracted in the [data](./data) folder." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python train.py \\\n", - "--dataset './data/ECG5000_TEST.arff' \\\n", - "--data_range 0 2627 \\\n", - "--minmax \\\n", - "--timesteps 140 \\\n", - "--encoder_dim 20 \\\n", - "--decoder_dim 40 \\\n", - "--output_activation 'sigmoid' \\\n", - "--dropout 0 \\\n", - "--learning_rate 0.005 \\\n", - "--loss 'mean_squared_error' \\\n", - "--epochs 100 \\\n", - "--batch_size 32 \\\n", - "--validation_split 0.2 \\\n", - "--print_progress \\\n", - "--save" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The plot below shows a typical prediction (*red line*) of an inlier (class 1) ECG compared to the original (*blue line*) after training the seq2seq-LSTM model." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![inlier_ecg](images/inlier_ecg.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "On the other hand, the model is not good at fitting ECG's from the other classes, as illustrated in the chart below:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![outlier_ecg](images/outlier_ecg.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The predictions in the above charts are made on ECG's the model has not seen before. The differences in scale are due to the sigmoid output layer and do not affect the prediction accuracy." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test using Kubernetes cluster on GCP or Minikube" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run the outlier detector as a model or a transformer. If you want to run the anomaly detector as a transformer, change the SERVICE_TYPE variable from MODEL to TRANSFORMER [here](./.s2i/environment), set MODEL = False and change ```OutlierSeq2SeqLSTM.py``` to:\n", - "\n", - "```python\n", - "from CoreSeq2SeqLSTM import CoreSeq2SeqLSTM\n", - "\n", - "class OutlierSeq2SeqLSTM(CoreSeq2SeqLSTM):\n", - " \"\"\" Outlier detection using a sequence-to-sequence (seq2seq) LSTM model.\n", - " \n", - " Parameters\n", - " ----------\n", - " threshold (float) : reconstruction error (mse) threshold used to classify outliers\n", - " reservoir_size (int) : number of observations kept in memory using reservoir sampling\n", - " \"\"\"\n", - " def __init__(self,threshold=0.003,reservoir_size=50000,model_name='seq2seq',load_path='./models/'):\n", - " \n", - " super().__init__(threshold=threshold,reservoir_size=reservoir_size,\n", - " model_name=model_name,load_path=load_path)\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pick Kubernetes cluster on GCP or Minikube." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MINIKUBE = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube start --memory 4096 \n", - "else:\n", - " !gcloud container clusters get-credentials standard-cluster-1 --zone europe-west1-b --project seldon-demos" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a cluster-wide cluster-admin role assigned to a service account named “default” in the namespace “kube-system”." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin \\\n", - "--serviceaccount=kube-system:default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add current context details to the configuration file in the seldon namespace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create tiller service account and give it a cluster-wide cluster-admin role." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl -n kube-system create sa tiller\n", - "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n", - "!helm init --service-account tiller" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status and deploy seldon/spartakus helm charts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/tiller-deploy -n kube-system" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usage_metrics.enabled=true --namespace seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status for seldon core." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install Ambassador API gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!helm install stable/ambassador --name ambassador --set crds.keep=false" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deployment.apps/ambassador" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If Minikube used: create docker image for outlier detector inside Minikube using s2i. Besides the transformer image and the demo specific model image, the general model image for the Seq2Seq LSTM outlier detector is also available from Docker Hub as ***seldonio/outlier-s2s-lstm-model:0.1***." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE & MODEL:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-s2s-lstm-model-demo:0.1\n", - "elif MINIKUBE:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-s2s-lstm-transformer:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install outlier detector helm charts and set *threshold* and *reservoir_size* hyperparameter values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " !helm install ../../../helm-charts/seldon-od-model \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set model.type=seq2seq \\\n", - " --set model.seq2seq.image.name=seldonio/outlier-s2s-lstm-model-demo:0.1 \\\n", - " --set model.seq2seq.threshold=0.002 \\\n", - " --set model.seq2seq.reservoir_size=50000 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set replicas=1\n", - "else:\n", - " !helm install ../../../helm-charts/seldon-od-transformer \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set outlierDetection.enabled=true \\\n", - " --set outlierDetection.name=outlier-s2s-lstm \\\n", - " --set outlierDetection.type=seq2seq \\\n", - " --set outlierDetection.seq2seq.image.name=seldonio/outlier-s2s-lstm-transformer:0.1 \\\n", - " --set outlierDetection.seq2seq.threshold=0.002 \\\n", - " --set outlierDetection.seq2seq.reservoir_size=50000 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set model.image.name=seldonio/outlier-s2s-lstm-model:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Ambassador\n", - "\n", - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import rest requests, load data and test requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from utils import get_payload, rest_request_ambassador, send_feedback_rest, ecg_data\n", - "\n", - "ecg_data, ecg_labels = ecg_data(dataset='TRAIN')\n", - "X = ecg_data[0,:].reshape(1,ecg_data.shape[1],1)\n", - "label = ecg_labels[0].reshape(1)\n", - "print(X.shape)\n", - "print(label.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test the rest requests with the generated data. It is important that the order of requests is respected. First we make predictions, then we get the \"true\" labels back using the feedback request. If we do not respect the order and eg keep making predictions without getting the feedback for each prediction, there will be a mismatch between the predicted and \"true\" labels. This will result in errors in the produced metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "request = get_payload(X)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the outlier detector is used as a transformer, the output of the anomaly detection is added as part of the metadata. If it is used as a model, we send model feedback to retrieve custom performance metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,label,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analytics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install the helm charts for prometheus and the grafana dashboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", - " --set grafana_prom_admin_password=password \\\n", - " --set persistence.enabled=false \\\n", - " --namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Grafana dashboard" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n", - "\n", - "Username : admin\n", - "\n", - "password : password (as set when starting seldon-core-analytics above)\n", - "\n", - "Import the outlier-detector-s2s-lstm dashboard from ../../../helm-charts/seldon-core-analytics/files/grafana/configs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run simulation\n", - "\n", - "- Sample random ECG from dataset.\n", - "- Get payload for the observation.\n", - "- Make a prediction.\n", - "- Send the \"true\" label with the feedback if the detector is run as a model.\n", - "\n", - "It is important that the prediction-feedback order is maintained. Otherwise there will be a mismatch between the predicted and \"true\" labels.\n", - "\n", - "View the progress on the grafana \"Outlier Detection\" dashboard. Most metrics need the outlier detector to be run as a model since they need model feedback." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "import time\n", - "n_requests = 100\n", - "n_samples, sample_length = ecg_data.shape\n", - "for i in range(n_requests):\n", - " idx = np.random.choice(n_samples)\n", - " X = ecg_data[idx,:].reshape(1,sample_length,1)\n", - " label = ecg_labels[idx].reshape(1)\n", - " request = get_payload(X)\n", - " response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")\n", - " if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,label,endpoint=\"localhost:8003\")\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube delete" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/components/outlier-detection/seq2seq-lstm/train.py b/components/outlier-detection/seq2seq-lstm/train.py deleted file mode 100644 index ee7a697e2e..0000000000 --- a/components/outlier-detection/seq2seq-lstm/train.py +++ /dev/null @@ -1,155 +0,0 @@ -import argparse -from keras.callbacks import ModelCheckpoint -import numpy as np -import pandas as pd -import pickle -import random -from scipy.io import arff - -from model import model - -np.random.seed(2018) -np.random.RandomState(2018) -random.seed(2018) - -# default args -DATASET = './data/ECG5000_TEST.arff' -SAVE_PATH = './models/' -MODEL_NAME = 'seq2seq' -DATA_RANGE = [0,2627] - -# data preprocessing -STANDARDIZED = False -MINMAX = False -CLIP = [99999] - -# architecture -TIMESTEPS = 140 # length of 1 ECG -ENCODER_DIM = [20] -DECODER_DIM = [40] -OUTPUT_ACTIVATION = 'sigmoid' - -# training -EPOCHS = 100 -BATCH_SIZE = 32 -LEARNING_RATE = .005 -LOSS = 'mean_squared_error' -DROPOUT = 0. -VALIDATION_SPLIT = 0.2 -SAVE = False -PRINT_PROGRESS = False -CONTINUE_TRAINING = False -LOAD_PATH = SAVE_PATH - -def train(model,X,args): - """ Train seq2seq-LSTM model. """ - - # clip data per feature - for col,clip in enumerate(args.clip): - X[:,:,col] = np.clip(X[:,:,col],-clip,clip) - - # apply scaling and save data preprocessing method - axis = (0,1) # scaling per feature over all observations - if args.standardized: - print('\nStandardizing data') - mu, sigma = np.mean(X,axis=axis), np.std(X,axis=axis) - X = (X - mu) / (sigma + 1e-10) - - with open(args.save_path + 'preprocess_' + args.model_name + '.pickle', 'wb') as f: - pickle.dump(['standardized',args.clip,axis,mu,sigma], f) - - if args.minmax: - print('\nMinmax scaling of data') - xmin, xmax = X.min(axis=axis), X.max(axis=axis) - min, max = 0, 1 - X = ((X - xmin) / (xmax - xmin)) * (max - min) + min - - with open(args.save_path + 'preprocess_' + args.model_name + '.pickle', 'wb') as f: - pickle.dump(['minmax',args.clip,axis,xmin,xmax,min,max], f) - - # define inputs - encoder_input_data = X - decoder_input_data = X - decoder_target_data = np.roll(X, -1, axis=1) # offset decoder_input_data by 1 across time axis - - # set training arguments - if args.print_progress: - verbose = 1 - else: - verbose = 0 - - kwargs = {} - kwargs['epochs'] = args.epochs - kwargs['batch_size'] = args.batch_size - kwargs['shuffle'] = True - kwargs['validation_split'] = args.validation_split - kwargs['verbose'] = verbose - - if args.save: # create callback - print('\nSave stuff') - checkpointer = ModelCheckpoint(filepath=args.save_path + args.model_name + '_weights.h5',verbose=0, - save_best_only=True,save_weights_only=True) - kwargs['callbacks'] = [checkpointer] - - # save model architecture - with open(args.save_path + args.model_name + '.pickle', 'wb') as f: - pickle.dump([X.shape[1],X.shape[2],args.encoder_dim, - args.decoder_dim,args.output_activation],f) - - model.fit([encoder_input_data, decoder_input_data], decoder_target_data, **kwargs) - -def run(args): - """ Load data, generate training batch, initiate and train model. """ - - print('\nLoad dataset') - data = arff.loadarff(args.dataset) - data = pd.DataFrame(data[0]) - data.drop(columns='target',inplace=True) - - print('\nGenerate training batch') - args.n_features = 1 # only 1 feature in the ECG dataset - X = data.values[args.data_range[0]:args.data_range[1],:] - X = np.reshape(X, (X.shape[0],X.shape[1],args.n_features)) - - print('\nInitiate outlier detector model') - s2s, enc, dec = model(args.n_features,encoder_dim=args.encoder_dim,decoder_dim=args.decoder_dim, - dropout=args.dropout,learning_rate=args.learning_rate,loss=args.loss, - output_activation=args.output_activation) - - if args.continue_training: - print('\nLoad pre-trained model') - s2s.load_weights(args.load_path + args.model_name + '_weights.h5') # load pretrained model weights - - if args.print_progress: - s2s.summary() - - print('\nTrain outlier detector') - train(s2s,X,args) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description="Train seq2seq-LSTM outlier detector.") - parser.add_argument('--dataset',type=str,choices=DATASET,default=DATASET) - parser.add_argument('--data_range',type=int,nargs=2,default=DATA_RANGE) - parser.add_argument('--timesteps',type=int,default=TIMESTEPS) - parser.add_argument('--encoder_dim',type=int,nargs='+',default=ENCODER_DIM) - parser.add_argument('--decoder_dim',type=int,nargs='+',default=DECODER_DIM) - parser.add_argument('--output_activation',type=str,default=OUTPUT_ACTIVATION) - parser.add_argument('--dropout',type=float,default=DROPOUT) - parser.add_argument('--learning_rate',type=float,default=LEARNING_RATE) - parser.add_argument('--loss',type=str,default=LOSS) - parser.add_argument('--validation_split',type=float,default=VALIDATION_SPLIT) - parser.add_argument('--epochs',type=int,default=EPOCHS) - parser.add_argument('--batch_size',type=int,default=BATCH_SIZE) - parser.add_argument('--clip',type=float,nargs='+',default=CLIP) - parser.add_argument('--standardized', default=STANDARDIZED, action='store_true') - parser.add_argument('--minmax', default=MINMAX, action='store_true') - parser.add_argument('--print_progress', default=PRINT_PROGRESS, action='store_true') - parser.add_argument('--save', default=SAVE, action='store_true') - parser.add_argument('--save_path',type=str,default=SAVE_PATH) - parser.add_argument('--load_path',type=str,default=LOAD_PATH) - parser.add_argument('--model_name',type=str,default=MODEL_NAME) - parser.add_argument('--continue_training', default=CONTINUE_TRAINING, action='store_true') - args = parser.parse_args() - - run(args) \ No newline at end of file diff --git a/components/outlier-detection/seq2seq-lstm/utils.py b/components/outlier-detection/seq2seq-lstm/utils.py deleted file mode 100644 index bbcb44cafb..0000000000 --- a/components/outlier-detection/seq2seq-lstm/utils.py +++ /dev/null @@ -1,91 +0,0 @@ -import collections -import json -import numpy as np -import pandas as pd -import requests -from scipy.io import arff -from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score, fbeta_score - -def ecg_data(dataset='TEST',data_range=None, outlier=[2,3,4,5]): - """ Return ECG dataset with outlier labels. """ - - data = arff.loadarff('./data/ECG5000_' + dataset + '.arff') - data = pd.DataFrame(data[0]) - data['target'] = data['target'].astype(int) - if data_range is None: - data_range = [0,data.shape[0]] - outlier_true = data['target'][data_range[0]:data_range[1]].isin(outlier).astype(int).values - data.drop(columns='target',inplace=True) - X = data.values[data_range[0]:data_range[1],:] - return X, outlier_true - -def flatten(x): - """ Flatten list. """ - if isinstance(x, collections.Iterable): - return [a for i in x for a in flatten(i)] - else: - return [x] - -def performance(y_true,y_pred,roll_window=100): - """ Return a confusion matrix and calculate rolling accuracy, precision, recall, F1 and F2 scores. """ - - # confusion matrix - cm = confusion_matrix(y_true,y_pred,labels=[0,1]) - tn, fp, fn, tp = cm.ravel() - - # total scores - acc_tot = accuracy_score(y_true,y_pred) - prec_tot = precision_score(y_true,y_pred) - rec_tot = recall_score(y_true,y_pred) - f1_tot = f1_score(y_true,y_pred) - f2_tot = fbeta_score(y_true,y_pred,beta=2) - - # rolling scores - y_true_roll = y_true[-roll_window:] - y_pred_roll = y_pred[-roll_window:] - acc_roll = accuracy_score(y_true_roll,y_pred_roll) - prec_roll = precision_score(y_true_roll,y_pred_roll) - rec_roll = recall_score(y_true_roll,y_pred_roll) - f1_roll = f1_score(y_true_roll,y_pred_roll) - f2_roll = fbeta_score(y_true_roll,y_pred_roll,beta=2) - - scores = [tn, fp, fn, tp, acc_tot, prec_tot, rec_tot, f1_tot, f2_tot, - acc_roll, prec_roll, rec_roll, f1_roll, f2_roll] - - return scores - -def outlier_stats(y_true,y_pred,roll_window=100): - """ Calculate number and percentage of predicted and labeled outliers. """ - - y_pred_roll = np.sum(y_pred[-roll_window:]) - y_true_roll = np.sum(y_true[-roll_window:]) - y_pred_tot = np.sum(y_pred) - y_true_tot = np.sum(y_true) - - return y_pred_roll, y_true_roll, y_pred_tot, y_true_tot - -def get_payload(arr): - features = ["x{}".format(str(i)) for i in range(arr.size)] - datadef = {"names":features,"ndarray":arr.tolist()} - payload = {"meta":{},"data":datadef} - return payload - -def rest_request_ambassador(deploymentName,namespace,request,endpoint="localhost:8003"): - response = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/predictions", - json=request) - print(response.status_code) - print(response.text) - return response.json() - -def send_feedback_rest(deploymentName,namespace,request,response,reward,truth,endpoint="localhost:8003"): - feedback = { - "request": request, - "response": response, - "reward": reward, - "truth": {"data":{"ndarray":truth.tolist()}} - } - ret = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/feedback", - json=feedback) - return diff --git a/components/outlier-detection/vae/.s2i/environment b/components/outlier-detection/vae/.s2i/environment deleted file mode 100644 index 273df4bc3c..0000000000 --- a/components/outlier-detection/vae/.s2i/environment +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=OutlierVAE -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/components/outlier-detection/vae/CoreVAE.py b/components/outlier-detection/vae/CoreVAE.py deleted file mode 100644 index 79d736435c..0000000000 --- a/components/outlier-detection/vae/CoreVAE.py +++ /dev/null @@ -1,182 +0,0 @@ -import logging -import numpy as np -import pickle -import random - -from model import model - -logger = logging.getLogger(__name__) - - -class CoreVAE(object): - """ Outlier detection using variational autoencoders (VAE). - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - reservoir_sampling : applies reservoir sampling to incoming data - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=10,reservoir_size=50000,model_name='vae',load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.reservoir_size = reservoir_size - self.batch = [] - self.N = 0 # total sample count up until now for reservoir sampling - self.nb_outliers = 0 - - # load model architecture parameters - with open(load_path + model_name + '.pickle', 'rb') as f: - n_features, hidden_layers, latent_dim, hidden_dim, output_activation = pickle.load(f) - - # instantiate model - self.vae = model(n_features,hidden_layers=hidden_layers,latent_dim=latent_dim, - hidden_dim=hidden_dim,output_activation=output_activation) - self.vae.load_weights(load_path + model_name + '_weights.h5') # load pretrained model weights - self.vae._make_predict_function() - - # load data preprocessing info - with open(load_path + 'preprocess_' + model_name + '.pickle', 'rb') as f: - preprocess = pickle.load(f) - self.preprocess, self.clip, self.axis = preprocess[:3] - if self.preprocess=='minmax': - self.xmin, self.xmax = preprocess[3:5] - self.min, self.max = preprocess[5:] - elif self.preprocess=='standardized': - self.mu, self.sigma = preprocess[3:] - - - def reservoir_sampling(self,X,update_stand=False): - """ Keep batch of data in memory using reservoir sampling. """ - for item in X: - self.N+=1 - if len(self.batch) < self.reservoir_size: - self.batch.append(item) - else: - s = int(random.random() * self.N) - if s < self.reservoir_size: - self.batch[s] = item - - if update_stand: - if self.preprocess=='minmax': - self.xmin = np.array(self.batch).min(axis=self.axis) - self.xmax = np.array(self.batch).max(axis=self.axis) - elif self.preprocess=='standardized': - self.mu = np.array(self.batch).mean(axis=self.axis) - self.sigma = np.array(self.batch).std(axis=self.axis) - return - - - def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) - - - def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X - - - def _get_preds(self, X): - """ Detect outliers if the reconstruction error is above the threshold. - - Parameters - ---------- - X : array-like - """ - - # clip data per feature - X = np.clip(X,[-c for c in self.clip],self.clip) - - if self.N < self.reservoir_size: - update_stand = False - else: - update_stand = True - - self.reservoir_sampling(X,update_stand=update_stand) - - # apply scaling - if self.preprocess=='minmax': - X_scaled = ((X - self.xmin) / (self.xmax - self.xmin)) * (self.max - self.min) + self.min - elif self.preprocess=='standardized': - X_scaled = (X - self.mu) / (self.sigma + 1e-10) - - # sample latent variables and calculate reconstruction errors - N = 10 - mse = np.zeros([X.shape[0],N]) - for i in range(N): - preds = self.vae.predict(X_scaled) - mse[:,i] = np.mean(np.power(X_scaled - preds, 2), axis=1) - self.mse = np.mean(mse, axis=1) - - # make prediction - self.prediction = np.array([1 if e > self.threshold else 0 for e in self.mse]).astype(int) - - return self.prediction - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return additional data as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - logger.info("Send feedback called") - return [] - - - def tags(self): - """ - Use predictions made within transform to add these as metadata - to the response. Tags will only be collected if the component is - used as an input-transformer. - """ - try: - return {"outlier-predictions": self.prediction_meta.tolist()} - except AttributeError: - logger.info("No metadata about outliers") - - - def metrics(self): - """ Return custom metrics averaged over the prediction batch. - """ - self.nb_outliers += np.sum(self.prediction) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":np.mean(self.prediction)} - mse = {"type":"GAUGE","key":"mse","value":np.mean(self.mse)} - nb_outliers = {"type":"GAUGE","key":"nb_outliers","value":int(self.nb_outliers)} - fraction_outliers = {"type":"GAUGE","key":"fraction_outliers","value":int(self.nb_outliers)/self.N} - obs = {"type":"GAUGE","key":"observation","value":self.N} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - return [is_outlier,mse,nb_outliers,fraction_outliers,obs,threshold] \ No newline at end of file diff --git a/components/outlier-detection/vae/OutlierVAE.py b/components/outlier-detection/vae/OutlierVAE.py deleted file mode 100644 index 7f92dcf866..0000000000 --- a/components/outlier-detection/vae/OutlierVAE.py +++ /dev/null @@ -1,119 +0,0 @@ -import numpy as np - -from CoreVAE import CoreVAE -from utils import flatten, performance, outlier_stats - - -class OutlierVAE(CoreVAE): - """ Outlier detection using variational autoencoders (VAE). - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - - def __init__(self,threshold=10,reservoir_size=50000,model_name='vae',load_path='./models/'): - - super().__init__(threshold=threshold,reservoir_size=reservoir_size, - model_name=model_name,load_path=load_path) - - self._predictions = [] - self._labels = [] - self._mse = [] - self.roll_window = 100 - self.metric = [float('nan') for i in range(18)] - - - def send_feedback(self,X,feature_names,reward,truth): - """ Return outlier labels as part of the feedback loop. - - Parameters - ---------- - X : array of the features sent in the original predict request - feature_names : array of feature names. May be None if not available. - reward (float): the reward - truth : array with correct value (optional) - """ - _ = super().send_feedback(X,feature_names,reward,truth) - - # historical reconstruction errors and predictions - self._mse.append(self.mse) - self._mse = flatten(self._mse) - self._predictions.append(self.prediction) - self._predictions = flatten(self._predictions) - - # target labels - self.label = truth - self._labels.append(self.label) - self._labels = flatten(self._labels) - - # performance metrics - scores = performance(self._labels,self._predictions,roll_window=self.roll_window) - stats = outlier_stats(self._labels,self._predictions,roll_window=self.roll_window) - - convert = flatten([scores,stats]) - metric = [] - for c in convert: # convert from np to native python type to jsonify - metric.append(np.asscalar(np.asarray(c))) - self.metric = metric - - return [] - - - def metrics(self): - """ Return custom metrics. - Printed with a delay of 1 prediction because the labels are returned in the feedback step. - """ - - if self.mse.shape[0]>1: - raise ValueError('Metrics can only handle single observations.') - - if self.N==1: - pred = float('nan') - err = float('nan') - y_true = float('nan') - else: - pred = int(self._predictions[-1]) - err = self._mse[-1] - y_true = int(self.label[0]) - - is_outlier = {"type":"GAUGE","key":"is_outlier","value":pred} - mse = {"type":"GAUGE","key":"mse","value":err} - obs = {"type":"GAUGE","key":"observation","value":self.N - 1} - threshold = {"type":"GAUGE","key":"threshold","value":self.threshold} - - label = {"type":"GAUGE","key":"label","value":y_true} - - accuracy_tot = {"type":"GAUGE","key":"accuracy_tot","value":self.metric[4]} - precision_tot = {"type":"GAUGE","key":"precision_tot","value":self.metric[5]} - recall_tot = {"type":"GAUGE","key":"recall_tot","value":self.metric[6]} - f1_score_tot = {"type":"GAUGE","key":"f1_tot","value":self.metric[7]} - f2_score_tot = {"type":"GAUGE","key":"f2_tot","value":self.metric[8]} - - accuracy_roll = {"type":"GAUGE","key":"accuracy_roll","value":self.metric[9]} - precision_roll = {"type":"GAUGE","key":"precision_roll","value":self.metric[10]} - recall_roll = {"type":"GAUGE","key":"recall_roll","value":self.metric[11]} - f1_score_roll = {"type":"GAUGE","key":"f1_roll","value":self.metric[12]} - f2_score_roll = {"type":"GAUGE","key":"f2_roll","value":self.metric[13]} - - true_negative = {"type":"GAUGE","key":"true_negative","value":self.metric[0]} - false_positive = {"type":"GAUGE","key":"false_positive","value":self.metric[1]} - false_negative = {"type":"GAUGE","key":"false_negative","value":self.metric[2]} - true_positive = {"type":"GAUGE","key":"true_positive","value":self.metric[3]} - - nb_outliers_roll = {"type":"GAUGE","key":"nb_outliers_roll","value":self.metric[14]} - nb_labels_roll = {"type":"GAUGE","key":"nb_labels_roll","value":self.metric[15]} - nb_outliers_tot = {"type":"GAUGE","key":"nb_outliers_tot","value":self.metric[16]} - nb_labels_tot = {"type":"GAUGE","key":"nb_labels_tot","value":self.metric[17]} - - return [is_outlier,mse,obs,threshold,label, - accuracy_tot,precision_tot,recall_tot,f1_score_tot,f2_score_tot, - accuracy_roll,precision_roll,recall_roll,f1_score_roll,f2_score_roll, - true_negative,false_positive,false_negative,true_positive, - nb_outliers_roll,nb_labels_roll,nb_outliers_tot,nb_labels_tot] \ No newline at end of file diff --git a/components/outlier-detection/vae/README.md b/components/outlier-detection/vae/README.md deleted file mode 100644 index 07bc4ecc70..0000000000 --- a/components/outlier-detection/vae/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Variational Auto-Encoder (VAE) Outlier Detector - -## Description - -[Anomaly or outlier detection](https://en.wikipedia.org/wiki/Anomaly_detection) has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The implemented VAE outlier detector aims to predict anomalies in tabular data. The VAE model can be trained in an unsupervised or semi-supervised way, which is helpful since labeled training data is often scarce. The outlier detector predicts whether the input features represent normal behaviour or not, dependent on a threshold level set by the user. - -## Implementation - -The architecture of the VAE is defined in ```model.py``` and the model is trained by running the ```train.py``` script. The ```OutlierVAE``` class loads a pre-trained model and makes predictions on new data. - -A detailed explanation of the implementation and usage of the Variational Auto-Encoder as an outlier detector can be found in the [VAE documentation](./doc.md). - -## Running on Seldon - -An end-to-end example running a VAE outlier detector on GCP or Minikube using Seldon to identify computer network intrusions is available [here](./outlier_vae.ipynb). - -Docker images to use the generic VAE outlier detector as a model or transformer can be found on Docker Hub: -* [seldonio/outlier-vae-model](https://hub.docker.com/r/seldonio/outlier-vae-model) -* [seldonio/outlier-vae-transformer](https://hub.docker.com/r/seldonio/outlier-vae-transformer) - -A model docker image specific for the demo is also available: -* [seldonio/outlier-vae-model-demo](https://hub.docker.com/r/seldonio/outlier-vae-model-demo) \ No newline at end of file diff --git a/components/outlier-detection/vae/__init__.py b/components/outlier-detection/vae/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/components/outlier-detection/vae/doc.md b/components/outlier-detection/vae/doc.md deleted file mode 100644 index d26290affc..0000000000 --- a/components/outlier-detection/vae/doc.md +++ /dev/null @@ -1,292 +0,0 @@ -# Variational Auto-Encoder Outlier (VAE) Algorithm Documentation - -The aim of this document is to explain the Variational Auto-Encoder algorithm in Seldon's outlier detection framework. - -First, we provide a high level overview of the algorithm and the use case, then we will give a detailed explanation of the implementation. - -## Overview - -Outlier detection has many applications, ranging from preventing credit card fraud to detecting computer network intrusions. The available data is typically unlabeled and detection needs to be done in real-time. The outlier detector can be used as a standalone algorithm, or to detect anomalies in the input data of another predictive model. - -The VAE outlier detection algorithm predicts whether the input features are an outlier or not, dependent on a threshold level set by the user. The algorithm needs to be pretrained first on a batch of -preferably- inliers. - -As observations arrive, the algorithm will: -- scale (standardize or minmax) the input features -- first encode, and then decode the input data in an attempt to reconstruct the initial observations -- compute a reconstruction error between the output of the decoder and the input data -- predict that the observation is an outlier if the error is larger than the threshold level - -## Why Variational Auto-Encoders? - -An Auto-Encoder is an algorithm that consists of 2 main building blocks: an encoder and a decoder. The encoder tries to find a compressed representation of the input data. The compressed data is then fed into the decoder, which aims to replicate the input data. Both the encoder and decoder are typically implemented with neural networks. The loss function to be minimized with stochastic gradient descent is a distance function between the input data and output of the decoder, and is called the reconstruction error. - -If we train the Auto-Encoder with inliers, it will be able to replicate new inlier data well with a low reconstruction error. However, if outliers are fed to the Auto-Encoder, the reconstruction error becomes large and we can classify the observation as an anomaly. - -A Variational Auto-Encoder adds constraints to the encoded representations of the input. The encodings are parameters of a probability distribution modeling the data. The decoder can then generate new data by sampling from the learned distribution. - -## Implementation - -### 1. Building the VAE model - -The VAE model definition in ```model.py``` takes 4 arguments that define the architecture: -- the number of features in the input -- the number of hidden layers used in the encoder and decoder -- the dimension of the latent variable -- the dimensions of each hidden layer - -``` python -def model(n_features, hidden_layers=1, latent_dim=2, hidden_dim=[], - output_activation='sigmoid', learning_rate=0.001): - """ Build VAE model. - - Arguments: - - n_features (int): number of features in the data - - hidden_layers (int): number of hidden layers used in encoder/decoder - - latent_dim (int): dimension of latent variable - - hidden_dim (list): list with dimension of each hidden layer - - output_activation (str): activation type for last dense layer in the decoder - - learning_rate (float): learning rate used during training - """ -``` - -First, the input data feeds in the encoder and is compressed by mapping it on the latent space which defines the probability distribution of the encodings: - -``` python - # encoder - inputs = Input(shape=(n_features,), name='encoder_input') - # define hidden layers - enc_hidden = Dense(hidden_dim[0], activation='relu', name='encoder_hidden_0')(inputs) - i = 1 - while i < hidden_layers: - enc_hidden = Dense(hidden_dim[i],activation='relu',name='encoder_hidden_'+str(i))(enc_hidden) - i+=1 - - z_mean = Dense(latent_dim, name='z_mean')(enc_hidden) - z_log_var = Dense(latent_dim, name='z_log_var')(enc_hidden) -``` - -We can then sample data from the latent space. - -``` python -def sampling(args): - """ Reparameterization trick by sampling from an isotropic unit Gaussian. - - Arguments: - - args (tensor): mean and log of variance of Q(z|X) - - Returns: - - z (tensor): sampled latent vector - """ - z_mean, z_log_var = args - batch = K.shape(z_mean)[0] - dim = K.int_shape(z_mean)[1] - epsilon = K.random_normal(shape=(batch, dim)) # by default, random_normal has mean=0 and std=1.0 - return z_mean + K.exp(0.5 * z_log_var) * epsilon # mean + stdev * eps -``` - -``` python - # reparametrization trick to sample z - z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var]) -``` - -The sampled data passes through the decoder which aims to reconstruct the input. - -``` python - # decoder - latent_inputs = Input(shape=(latent_dim,), name='z_sampling') - # define hidden layers - dec_hidden = Dense(hidden_dim[-1], activation='relu', name='decoder_hidden_0')(latent_inputs) - - i = 2 - while i < hidden_layers+1: - dec_hidden = Dense(hidden_dim[-i],activation='relu',name='decoder_hidden_'+str(i-1))(dec_hidden) - i+=1 - - outputs = Dense(n_features, activation=output_activation, name='decoder_output')(dec_hidden) -``` - -The loss function is the sum of the reconstruction error and the KL-divergence. While the reconstruction error quantifies how well we can recreate the input data, the KL-divergence measures how close the latent representation is to the unit Gaussian distribution. This trade-off is important because we want our encodings to parameterize a probability distribution from which we can sample data. - -``` python - # define VAE loss, optimizer and compile model - reconstruction_loss = mse(inputs, outputs) - reconstruction_loss *= n_features - kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) - kl_loss = K.sum(kl_loss, axis=-1) - kl_loss *= -0.5 - vae_loss = K.mean(reconstruction_loss + kl_loss) - vae.add_loss(vae_loss) -``` - -### 2. Training the model - -The VAE model can be trained on a batch of inliers by running the ```train.py``` script with the desired hyperparameters: - -``` python -!python train.py \ ---dataset 'kddcup99' \ ---samples 50000 \ ---keep_cols "$cols_str" \ ---hidden_layers 1 \ ---latent_dim 2 \ ---hidden_dim 9 \ ---output_activation 'sigmoid' \ ---clip 999999 \ ---standardized \ ---epochs 10 \ ---batch_size 32 \ ---learning_rate 0.001 \ ---print_progress \ ---model_name 'vae' \ ---save \ ---save_path './models/' -``` - -The model weights and hyperparameters are saved in the folder specified by "save_path". - -### 3. Making predictions - -In order to make predictions, which can then be served by Seldon Core, the pre-trained model weights and hyperparameters are loaded when defining an OutlierVAE object. The "threshold" argument defines above which reconstruction error a sample is classified as an outlier. The threshold is a key hyperparameter and needs to be picked carefully for each application. The OutlierVAE class inherits from the CoreVAE class in ```CoreVAE.py```. - -```python -class CoreVAE(object): - """ Outlier detection using variational autoencoders (VAE). - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - reservoir_sampling : applies reservoir sampling to incoming data - predict : detect and return outliers - transform_input : detect outliers and return input features - send_feedback : add target labels as part of the feedback loop - tags : add metadata for input transformer - metrics : return custom metrics - """ - - def __init__(self,threshold=10,reservoir_size=50000,model_name='vae',load_path='./models/'): - - logger.info("Initializing model") - self.threshold = threshold - self.reservoir_size = reservoir_size - self.batch = [] - self.N = 0 # total sample count up until now for reservoir sampling - self.nb_outliers = 0 - - # load model architecture parameters - with open(load_path + model_name + '.pickle', 'rb') as f: - n_features, hidden_layers, latent_dim, hidden_dim, output_activation = pickle.load(f) - - # instantiate model - self.vae = model(n_features,hidden_layers=hidden_layers,latent_dim=latent_dim, - hidden_dim=hidden_dim,output_activation=output_activation) - self.vae.load_weights(load_path + model_name + '_weights.h5') # load pretrained model weights - self.vae._make_predict_function() - - # load data preprocessing info - with open(load_path + 'preprocess_' + model_name + '.pickle', 'rb') as f: - preprocess = pickle.load(f) - self.preprocess, self.clip, self.axis = preprocess[:3] - if self.preprocess=='minmax': - self.xmin, self.xmax = preprocess[3:5] - self.min, self.max = preprocess[5:] - elif self.preprocess=='standardized': - self.mu, self.sigma = preprocess[3:] -``` - -``` python -class OutlierVAE(CoreVAE): - """ Outlier detection using variational autoencoders (VAE). - - Parameters - ---------- - threshold (float) : reconstruction error (mse) threshold used to classify outliers - reservoir_size (int) : number of observations kept in memory using reservoir sampling - - Functions - ---------- - send_feedback : add target labels as part of the feedback loop - metrics : return custom metrics - """ - - def __init__(self,threshold=10,reservoir_size=50000,model_name='vae',load_path='./models/'): - - super().__init__(threshold=threshold,reservoir_size=reservoir_size, - model_name=model_name,load_path=load_path) -``` - -The actual outlier detection is done by the ```_get_preds``` method which is invoked by ```predict``` or ```transform_input``` dependent on whether the detector is defined as respectively a model or a transformer. - -```python -def predict(self, X, feature_names): - """ Return outlier predictions. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as a model") - return self._get_preds(X) -``` - -```python -def transform_input(self, X, feature_names): - """ Transform the input. - Used when the outlier detector sits on top of another model. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - logger.info("Using component as an outlier-detector transformer") - self.prediction_meta = self._get_preds(X) - return X -``` - -In ```_get_preds```, the observations are first clipped. If the number of observations fed to the outlier detector up until now is at least equal to the defined reservoir size, the feature-wise scaling parameters are updated using the observations in the reservoir. The reservoir is updated each observation using reservoir sampling. The input data is then scaled using either standardization or minmax scaling. - -``` python - # clip data per feature - X = np.clip(X,[-c for c in self.clip],self.clip) - - if self.N < self.reservoir_size: - update_stand = False - else: - update_stand = True - - self.reservoir_sampling(X,update_stand=update_stand) - - # apply scaling - if self.preprocess=='minmax': - X_scaled = ((X - self.xmin) / (self.xmax - self.xmin)) * (self.max - self.min) + self.min - elif self.preprocess=='standardized': - X_scaled = (X - self.mu) / (self.sigma + 1e-10) -``` - -We then make multiple predictions for an observation by sampling N times from the latent space. The mean squared error between the input data and output of the decoder is averaged across the N samples. If this value is above the threshold, an outlier is predicted. - -``` python - # sample latent variables and calculate reconstruction errors - N = 10 - mse = np.zeros([X.shape[0],N]) - for i in range(N): - preds = self.vae.predict(X_scaled) - mse[:,i] = np.mean(np.power(X_scaled - preds, 2), axis=1) - self.mse = np.mean(mse, axis=1) - - # make prediction - self.prediction = np.array([1 if e > self.threshold else 0 for e in self.mse]).astype(int) -``` - -## References - -Diederik P. Kingma and Max Welling. Auto-Encoding Variational Bayes. ICLR 2014. -- https://arxiv.org/pdf/1312.6114.pdf - -Francois Chollet. Building Autoencoders in Keras. -- https://blog.keras.io/building-autoencoders-in-keras.html \ No newline at end of file diff --git a/components/outlier-detection/vae/model.py b/components/outlier-detection/vae/model.py deleted file mode 100644 index e54c61e65f..0000000000 --- a/components/outlier-detection/vae/model.py +++ /dev/null @@ -1,92 +0,0 @@ -from keras.layers import Lambda, Input, Dense -from keras.models import Model -from keras.losses import mse -from keras import backend as K -from keras.optimizers import Adam -import numpy as np - -def sampling(args): - """ Reparameterization trick by sampling from an isotropic unit Gaussian. - - Arguments: - - args (tensor): mean and log of variance of Q(z|X) - - Returns: - - z (tensor): sampled latent vector - """ - z_mean, z_log_var = args - batch = K.shape(z_mean)[0] - dim = K.int_shape(z_mean)[1] - epsilon = K.random_normal(shape=(batch, dim)) # by default, random_normal has mean=0 and std=1.0 - return z_mean + K.exp(0.5 * z_log_var) * epsilon # mean + stdev * eps - -def model(n_features, hidden_layers=1, latent_dim=2, hidden_dim=[], - output_activation='sigmoid', learning_rate=0.001): - """ Build VAE model. - - Arguments: - - n_features (int): number of features in the data - - hidden_layers (int): number of hidden layers used in encoder/decoder - - latent_dim (int): dimension of latent variable - - hidden_dim (list): list with dimension of each hidden layer - - output_activation (str): activation type for last dense layer in the decoder - - learning_rate (float): learning rate used during training - """ - - # set dimensions hidden layers - if hidden_dim==[]: - i = 0 - dim = n_features - while i < hidden_layers: - hidden_dim.append(int(np.max([dim/2,2]))) - dim/=2 - i+=1 - - # VAE = encoder + decoder - # encoder - inputs = Input(shape=(n_features,), name='encoder_input') - # define hidden layers - enc_hidden = Dense(hidden_dim[0], activation='relu', name='encoder_hidden_0')(inputs) - i = 1 - while i < hidden_layers: - enc_hidden = Dense(hidden_dim[i],activation='relu',name='encoder_hidden_'+str(i))(enc_hidden) - i+=1 - - z_mean = Dense(latent_dim, name='z_mean')(enc_hidden) - z_log_var = Dense(latent_dim, name='z_log_var')(enc_hidden) - # reparametrization trick to sample z - z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var]) - # instantiate encoder model - encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder') - - # decoder - latent_inputs = Input(shape=(latent_dim,), name='z_sampling') - # define hidden layers - dec_hidden = Dense(hidden_dim[-1], activation='relu', name='decoder_hidden_0')(latent_inputs) - - i = 2 - while i < hidden_layers+1: - dec_hidden = Dense(hidden_dim[-i],activation='relu',name='decoder_hidden_'+str(i-1))(dec_hidden) - i+=1 - - outputs = Dense(n_features, activation=output_activation, name='decoder_output')(dec_hidden) - # instantiate decoder model - decoder = Model(latent_inputs, outputs, name='decoder') - - # instantiate VAE model - outputs = decoder(encoder(inputs)[2]) - vae = Model(inputs, outputs, name='vae') - - # define VAE loss, optimizer and compile model - reconstruction_loss = mse(inputs, outputs) - reconstruction_loss *= n_features - kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) - kl_loss = K.sum(kl_loss, axis=-1) - kl_loss *= -0.5 - vae_loss = K.mean(reconstruction_loss + kl_loss) - vae.add_loss(vae_loss) - - optimizer = Adam(lr=learning_rate) - vae.compile(optimizer=optimizer) - - return vae \ No newline at end of file diff --git a/components/outlier-detection/vae/models/preprocess_vae.pickle b/components/outlier-detection/vae/models/preprocess_vae.pickle deleted file mode 100644 index 0496d54491234f7008a3965150e7fadd6e331681..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 518 zcmZo*jxA)+h~QyhU??t0%u7iuO3AEBO(|rIEo5?c)N82&N-!jPGbHDg<`z`yCFd8V z>gAT^lw>9r6(v@3#TTU}=jRod6qP2Ia1}B`RB?ff0m&A!cr&yXvN|&+6|%J#vIiA% zXm~Sv3$+$>!|0c}aMDCG8L@Mdf;@6fW*gxq=u}jD4ppS)v3Pqqk7R}%Vx>@$3V^iG=Q-?L~cUIqspJ%To z(S5f%ZLWP|O6EMJjI;KB2RL2?TA#JgysfZzX~}l`*~j*MJfODEzA?zK*J0yD`<^$4 z+&S(?{*>qE?Thx}^BJEX`n|$_f!tO; pk4baw(*(VP3KF~RE#;>5S6)A3e@sMOz18@PeW6%Mp?FfN9ss$U(VzeT diff --git a/components/outlier-detection/vae/models/vae.pickle b/components/outlier-detection/vae/models/vae.pickle deleted file mode 100644 index 7dc3774cc18206c64db179c77542a30a858ee0dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34 pcmZo*jxA)+@D}oB^k#}JWc21tj9_PAU?|Q^&&|(FDP&630|17O2t@z@ diff --git a/components/outlier-detection/vae/models/vae_weights.h5 b/components/outlier-detection/vae/models/vae_weights.h5 deleted file mode 100644 index a6d56ac30b66aec04b79b7279fc6d50109283dd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18864 zcmeHO2~-qE6CO@69Eum0{B9&7DheV|gq;`pmG^l3?N55@D`#dilXrX zF(zoDqLPTZQ@!4T`TrnBMWe?$p0)>*f)=5vgyv}r1uaIOQMQ?r`i?}u zNJpq}Bscnx89tm*NK*ZjlH2MP7}Qs23T+7ipWz;U%J~M8N!t;6ir5)pnNZ1;n5m($ zL6MT_GvXDMglDFPs_Ip?sFZc2H#9>{iM^`ZwGk7n@D-)S8Zib6>y8RZR`eOT@*b{4w>>%sAMW*p-`*zROP?W zgWwrLnPF58O`JtpXTPd)NEGeO>C;olD7sf&0_4h`8(&(A zUX=Z*`+LWBg6v#!i{S4})BaB6G^A}JpziO)^q4~ZhImkW_iI?)VvhQIlG3>q*E-VQ zQ~xnm_>Vc07#bNK5&zWTO+_4W4|m4dka4afj!ykw(Dv--Kl7h!R>HjoWWrhsFMHO{ z=(y$9f}FYN*}j@Cdk-5mLZy-nhvF$b5HL)9s3a~XHY_S8QP4q`Th)4?sB#*h_5Xb{jKpg}-`fCd2#0vZG~2xt&^B@h_<{wPl^MO0EmMl;t% zD9|`S9^p^93iULumqFrrRL;jjXpf+A3Zl=C#8If6dR&Ewz4oNJ{6&UHm?9ys{d_eO z_-6>H$1{mrF{#8~s5Y7xh=`mzHB=JhNb5fdiVh7?1nNHtii!yjN(hN14b-nhR8Wo5 zxS+a!8Zjj#A7HG?U#Q13mH7*$OVjuz4ZBjRK28TkLO~K5Y>7oVML6X0b`azTCBV$d!hS8hKUfw zUeafC>s8sC!a!);{Ur@c)6FY_fcknRBI9zQ1;SIeCoDr9Qfb%9i>`A5*Ze1a9-($u zsP>$pp5G9>Sm8Z`1l_dX48rHXE!2xh*Y5or%0&=9)qx27ONfA@c6}aJ)bGU;qj=xy z{$jbd)>?CEKAUwr32L0IQMYLa!J%(3tn}Ro@t^1NpN02kS3TyyUY5W~w6@Gn72t1{u6?YhwOY$CsP%qleNO$L72<&Of! z&VjBOTAbaTQ)uRy%Mg&X2JZLLM)_W!ArG5l$l-Hym>;i)Z2oXT!Hg4H`k@^<`@{iU zD=T@E)1&bZdir>5R|))5;o9d>K0;^9kSeT~p+h z9><3`UFXm4T!V~)es|sL-V^22rNjCmgRqYL49b%3fSj8ZC@R?vi6ysT;Hjn39pO4y zukH)jl5B*E!j3@Cig;M@FawSE?FhD#E$HAc*>Jmt!6mlY=)AmyAG+cx9pesNr-5meo%=F6_{h5ORAuxMQroY+(V)9zj2zi~=K77v$!qg66^ z>>0uurU3l6dp2JgqJwYog}l7VMEsWPK5>M344=Ni3^BRivu-K7kmry4`D#6DbfE4K zBuI+=mbF7Bj&QbOXE;Df0A`7U5%^)F0h-bPod05(J1`k1z7XvT1ZGLWV;!C z%hryW5B_V5S$S^@{Gi|z(oWou4$NGRejm0QwtT#iJzh7Dm-a7)p(CrLk!Kgeg`_&j zSQf|ayP1i?AC2b!S36Cdrr(XL{(dZ96I?1~w^ksV${=y~ifhniqlvh`_btd@w27UN zu#Jzu_yupEjoIRa!@#dvj`%(Wh?Vt%;n$qu^fq61PjNeO?@_<7UKKgWXvb>4k3)Ik zIe9Hg@4o>OZO%Z2{t$k_u^XsL{xyHh%$&Q>YXKT>V23x3n=2KYoQCqaO`x~r2m<+I z-plwFO8hE=o#>D7;+u#0?~YqSO20hTVE;(|u;n7D^kM++H~up;YceD2Tx-eQSsB44 zdrU&2F`nGf5tX>0`Z3zLIowV6-Xi?7Q2=*gW-{ttRwCA)l8dukE^z^2{x~n{4mu2{ zxzuyNbBQjDyJgy4j5VBn5uEdTODu3P42Zk1Ch`oY{%mbQKpl>WFC8@xM< zE%$|?Ma+giR6Y6c9#m;5JjVRz?ocdp7{*_to$c?zA{Zy+l! zb_VT(a;TEoe%;ysa;+~rP3V8P`j(hdKy;f|Nu%dEQo zg`GVX;;LI|c;O>2*^@1{vPmxI(4P~sktoa;pV++%uPWS%yW3Tub~7=au=jIVd1XF7 z&hH5;_kMz6hGpaRk`+)hay7ndD~BPKwXn~r1nZ4CioI4Yf;&&d*r~1*j<4H`D~gWb zZBd<_m<$vz7p=*(MJ%t_#~K(bj82TH^%$Qa&hmUG2ZZX zAgmn#(#M0k;QJ;gIFrI9+{gm~ctge}95Tj*^VE&OxtBJBnUgj?yf6UE^(s+H+-7dr zf%Bj@co5!kek=F({w>_235Iw^)>d2+yc6rD|BBb^JjQ-z5BbOK4f&C_5`4@p6S{qz z4)ZGQa9^)c++T7-?6`9mUi9S_ELv`eudJxSR<^Y`yCxYDt=i%B=K6SaNEt9%Jp5Q` z1J2#og29q}u+2IIm|(jtp-zXd`?Vi(EhTS#&`%%;Pf0#aiE7?b-v~Jd`Ix{ zX)xu+_(U{lZ*8Wj_psE@70sQu=?kg!IlQ&Utu%+LGz%^b6+v+~io#5lszyurP6Vaf(wV9^& z3H3NtbNhtdU}2;Uzx~!8y+;45fvXZ&ToW_r?7tiV^>s=_=9@w6hfcq>p5Lh5*%7;?>zukB({Ziy6h=hlWQ4!wMMv7UsCr&K z>75tPpWn(feT34M?$qJG%c{H(3dx2gS>A$CFWX|4TMSk)LN$A|Xxd`j&zb=dRU+iGod`vhJ8 XC~f&V7~Ds|xZxMoIBxv^jjR6+kA6$@ diff --git a/components/outlier-detection/vae/outlier_vae.ipynb b/components/outlier-detection/vae/outlier_vae.ipynb deleted file mode 100644 index 7290ac7c7e..0000000000 --- a/components/outlier-detection/vae/outlier_vae.ipynb +++ /dev/null @@ -1,622 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# VAE (variational autoencoder) outlier detector deployment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap a keras VAE python model for use as a prediction microservice in seldon-core and deploy on seldon-core running on minikube or a Kubernetes cluster using GCP." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- [helm](https://github.com/helm/helm)\n", - "- [minikube](https://github.com/kubernetes/minikube)\n", - "- [s2i](https://github.com/openshift/source-to-image) >= 1.1.13\n", - "\n", - "python packages:\n", - "- keras: pip install keras\n", - "- tensorflow: https://www.tensorflow.org/install/pip\n", - "- scikit-learn: pip install scikit-learn --> 0.20.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Task" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The outlier detector needs to detect computer network intrusions using TCP dump data for a local-area network (LAN) simulating a typical U.S. Air Force LAN. A connection is a sequence of TCP packets starting and ending at some well defined times, between which data flows to and from a source IP address to a target IP address under some well defined protocol. Each connection is labeled as either normal, or as an attack. \n", - "\n", - "There are 4 types of attacks in the dataset:\n", - "- DOS: denial-of-service, e.g. syn flood;\n", - "- R2L: unauthorized access from a remote machine, e.g. guessing password;\n", - "- U2R: unauthorized access to local superuser (root) privileges;\n", - "- probing: surveillance and other probing, e.g., port scanning.\n", - " \n", - "The dataset contains about 5 million connection records.\n", - "\n", - "There are 3 types of features:\n", - "- basic features of individual connections, e.g. duration of connection\n", - "- content features within a connection, e.g. number of failed log in attempts\n", - "- traffic features within a 2 second window, e.g. number of connections to the same host as the current connection\n", - "\n", - "The outlier detector is only using the continuous (18 out of 41) features." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train locally\n", - "\n", - "Train on small dataset of normal traffic." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# define columns to keep\n", - "cols=['srv_count','serror_rate','srv_serror_rate','rerror_rate',\n", - " 'srv_rerror_rate','same_srv_rate','diff_srv_rate',\n", - " 'srv_diff_host_rate','dst_host_count','dst_host_srv_count',\n", - " 'dst_host_same_srv_rate','dst_host_diff_srv_rate',\n", - " 'dst_host_same_src_port_rate','dst_host_srv_diff_host_rate',\n", - " 'dst_host_serror_rate','dst_host_srv_serror_rate',\n", - " 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target']\n", - "cols_str = str(cols)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python train.py \\\n", - "--dataset 'kddcup99' \\\n", - "--samples 50000 \\\n", - "--keep_cols \"$cols_str\" \\\n", - "--hidden_layers 1 \\\n", - "--latent_dim 2 \\\n", - "--hidden_dim 9 \\\n", - "--output_activation 'sigmoid' \\\n", - "--clip 999999 \\\n", - "--standardized \\\n", - "--epochs 10 \\\n", - "--batch_size 32 \\\n", - "--learning_rate 0.001 \\\n", - "--print_progress \\\n", - "--model_name 'vae' \\\n", - "--save \\\n", - "--save_path './models/'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test using Kubernetes cluster on GCP or Minikube" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run the outlier detector as a model or a transformer. If you want to run the anomaly detector as a transformer, change the SERVICE_TYPE variable from MODEL to TRANSFORMER [here](./.s2i/environment), set MODEL = False and change ```OutlierVAE.py``` to:\n", - "\n", - "```python\n", - "from CoreVAE import CoreVAE\n", - "\n", - "class OutlierVAE(CoreVAE):\n", - " \"\"\" Outlier detection using variational autoencoders (VAE).\n", - " \n", - " Parameters\n", - " ----------\n", - " threshold (float) : reconstruction error (mse) threshold used to classify outliers\n", - " reservoir_size (int) : number of observations kept in memory using reservoir sampling\n", - " \"\"\"\n", - " \n", - " def __init__(self,threshold=10,reservoir_size=50000,model_name='vae',load_path='./models/'):\n", - " \n", - " super().__init__(threshold=threshold,reservoir_size=reservoir_size,\n", - " model_name=model_name,load_path=load_path)\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Pick Kubernetes cluster on GCP or Minikube." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MINIKUBE = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube start --memory 4096\n", - "else:\n", - " !gcloud container clusters get-credentials standard-cluster-1 --zone europe-west1-b --project seldon-demos" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a cluster-wide cluster-admin role assigned to a service account named “default” in the namespace “kube-system”." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin \\\n", - "--serviceaccount=kube-system:default" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Add current context details to the configuration file in the seldon namespace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create tiller service account and give it a cluster-wide cluster-admin role." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl -n kube-system create sa tiller\n", - "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n", - "!helm init --service-account tiller" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status and deploy seldon/spartakus helm charts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/tiller-deploy -n kube-system" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usage_metrics.enabled=true --namespace seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check deployment rollout status for seldon core." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install Ambassador API gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!helm install stable/ambassador --name ambassador --set crds.keep=false" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deployment.apps/ambassador" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If Minikube used: create docker image for outlier detector inside Minikube using s2i. Besides the transformer image and the demo specific model image, the general model image for the VAE outlier detector is also available from Docker Hub as ***seldonio/outlier-vae-model:0.1***." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "if MINIKUBE & MODEL:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-vae-model-demo:0.1\n", - "elif MINIKUBE:\n", - " !eval $(minikube docker-env) && \\\n", - " s2i build . seldonio/seldon-core-s2i-python3:0.4 seldonio/outlier-vae-transformer:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install outlier detector helm charts either as a model or transformer and set *threshold* and *reservoir_size* hyperparameter values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " !helm install ../../../helm-charts/seldon-od-model \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set model.type=vae \\\n", - " --set model.vae.image.name=seldonio/outlier-vae-model-demo:0.1 \\\n", - " --set model.vae.threshold=10 \\\n", - " --set model.vae.reservoir_size=50000 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set replicas=1\n", - "else:\n", - " !helm install ../../../helm-charts/seldon-od-transformer \\\n", - " --name outlier-detector \\\n", - " --namespace=seldon \\\n", - " --set outlierDetection.enabled=true \\\n", - " --set outlierDetection.name=outlier-vae \\\n", - " --set outlierDetection.type=vae \\\n", - " --set outlierDetection.vae.image.name=seldonio/outlier-vae-transformer:0.1 \\\n", - " --set outlierDetection.vae.threshold=10 \\\n", - " --set outlierDetection.vae.reservoir_size=50000 \\\n", - " --set oauth.key=oauth-key \\\n", - " --set oauth.secret=oauth-secret \\\n", - " --set model.image.name=seldonio/mock_classifier:1.0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Ambassador\n", - "\n", - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Import rest requests, load data and test requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from utils import get_payload, rest_request_ambassador, send_feedback_rest, get_kdd_data, generate_batch\n", - "\n", - "data = get_kdd_data(keep_cols=cols,percent10=True) # load dataset\n", - "print(data.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate a random batch from the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "samples = 1\n", - "fraction_outlier = 0.\n", - "X, labels = generate_batch(data,samples,fraction_outlier)\n", - "print(X.shape)\n", - "print(labels.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test the rest requests with the generated data. It is important that the order of requests is respected. First we make predictions, then we get the \"true\" labels back using the feedback request. If we do not respect the order and eg keep making predictions without getting the feedback for each prediction, there will be a mismatch between the predicted and \"true\" labels. This will result in errors in the produced metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "request = get_payload(X)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the outlier detector is used as a transformer, the output of the anomaly detection is added as part of the metadata. If it is used as a model, we send model feedback to retrieve custom performance metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analytics" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install the helm charts for prometheus and the grafana dashboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", - " --set grafana_prom_admin_password=password \\\n", - " --set persistence.enabled=false \\\n", - " --namespace seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Port forward Grafana dashboard" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Run command in terminal:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```\n", - "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n", - "\n", - "Username : admin\n", - "\n", - "password : password (as set when starting seldon-core-analytics above)\n", - "\n", - "Import the outlier-detector-vae dashboard from ../../../helm-charts/seldon-core-analytics/files/grafana/configs." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run simulation\n", - "\n", - "- Sample random network intrusion data with a certain outlier probability.\n", - "- Get payload for the observation.\n", - "- Make a prediction.\n", - "- Send the \"true\" label with the feedback if the detector is run as a model.\n", - "\n", - "It is important that the prediction-feedback order is maintained. Otherwise there will be a mismatch between the predicted and \"true\" labels.\n", - "\n", - "View the progress on the grafana \"Outlier Detection\" dashboard. Most metrics need the outlier detector to be run as a model since they need model feedback." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "import time\n", - "n_requests = 100\n", - "samples = 1\n", - "for i in range(n_requests):\n", - " fraction_outlier = .1\n", - " X, labels = generate_batch(data,samples,fraction_outlier)\n", - " request = get_payload(X)\n", - " response = rest_request_ambassador(\"outlier-detector\",\"seldon\",request,endpoint=\"localhost:8003\")\n", - " if MODEL:\n", - " send_feedback_rest(\"outlier-detector\",\"seldon\",request,response,0,labels,endpoint=\"localhost:8003\")\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if MINIKUBE:\n", - " !minikube delete" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/components/outlier-detection/vae/requirements.txt b/components/outlier-detection/vae/requirements.txt deleted file mode 100644 index dcf2c1bcef..0000000000 --- a/components/outlier-detection/vae/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -keras==2.2.2 -tensorflow==1.15.2 -numpy==1.14.5 -argparse==1.1 -pandas==0.23.4 -scikit-learn==0.19.1 -requests>=2.20.0 diff --git a/components/outlier-detection/vae/train.py b/components/outlier-detection/vae/train.py deleted file mode 100644 index 333c1f0501..0000000000 --- a/components/outlier-detection/vae/train.py +++ /dev/null @@ -1,147 +0,0 @@ -import argparse -from keras.callbacks import ModelCheckpoint -import numpy as np -import pickle -import random - -from model import model -from utils import get_kdd_data, generate_batch - -np.random.seed(2018) -np.random.RandomState(2018) -random.seed(2018) - -# default args -DATASET = 'kddcup99' -SAMPLES = 50000 -COLS = str(['srv_count','serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate', - 'same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count', - 'dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate', - 'dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target']) -MODEL_NAME = 'vae' -SAVE_PATH = './models/' - -# data preprocessing -STANDARDIZED = False -MINMAX = False -CLIP = [99999] - -# architecture -HIDDEN_LAYERS = 2 -LATENT_DIM = 2 -HIDDEN_DIM = [15,7] -OUTPUT_ACTIVATION = 'sigmoid' - -# training -EPOCHS = 20 -BATCH_SIZE = 32 -LEARNING_RATE = .001 -SAVE = False -PRINT_PROGRESS = False -CONTINUE_TRAINING = False -LOAD_PATH = SAVE_PATH - -def train(model,X,args): - """ Train VAE. """ - - # clip data per feature - X = np.clip(X,[-c for c in args.clip],args.clip) - - # apply scaling and save data preprocessing method - axis = 0 - if args.standardized: - print('\nStandardizing data') - mu, sigma = np.mean(X,axis=axis), np.std(X,axis=axis) - X = (X - mu) / (sigma + 1e-10) - - with open(args.save_path + 'preprocess_' + args.model_name + '.pickle', 'wb') as f: - pickle.dump(['standardized',args.clip,axis,mu,sigma], f) - - if args.minmax: - print('\nMinmax scaling of data') - xmin, xmax = X.min(axis=axis), X.max(axis=axis) - min, max = 0, 1 - X = ((X - xmin) / (xmax - xmin)) * (max - min) + min - - with open(args.save_path + 'preprocess_' + args.model_name + '.pickle', 'wb') as f: - pickle.dump(['minmax',args.clip,axis,xmin,xmax,min,max], f) - - # set training arguments - if args.print_progress: - verbose = 1 - else: - verbose = 0 - - kwargs = {} - kwargs['epochs'] = args.epochs - kwargs['batch_size'] = args.batch_size - kwargs['shuffle'] = True - kwargs['validation_data'] = (X,None) - kwargs['verbose'] = verbose - - if args.save: # create callback - checkpointer = ModelCheckpoint(filepath=args.save_path + args.model_name + '_weights.h5',verbose=0, - save_best_only=True,save_weights_only=True) - kwargs['callbacks'] = [checkpointer] - - # save model architecture - with open(args.save_path + args.model_name + '.pickle', 'wb') as f: - pickle.dump([X.shape[1],args.hidden_layers,args.latent_dim, - args.hidden_dim,args.output_activation],f) - - model.fit(X,**kwargs) - -def run(args): - """ Load data, generate training batch, initiate model and train VAE. """ - - print('\nLoad dataset') - if args.dataset=='kddcup99': - keep_cols = args.keep_cols[1:-1].replace("'","").replace(" ","").split(",") - data = get_kdd_data(keep_cols=keep_cols) - else: - raise ValueError('Only "kddcup99" dataset supported.') - - print('\nGenerate training batch') - X, _ = generate_batch(data,args.samples,0.) - - print('\nInitiate outlier detector model') - n_features = data.shape[1]-1 # nb of features - vae = model(n_features,hidden_layers=args.hidden_layers,latent_dim=args.latent_dim,hidden_dim=args.hidden_dim, - output_activation=args.output_activation,learning_rate=args.learning_rate) - - if args.continue_training: - print('\nLoad pre-trained model') - vae.load_weights(args.load_path + args.model_name + '_weights.h5') # load pretrained model weights - - if args.print_progress: - vae.summary() - - print('\nTrain outlier detector') - train(vae,X,args) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description="Train VAE outlier detector.") - parser.add_argument('--dataset',type=str,choices=DATASET,default=DATASET) - parser.add_argument('--samples',type=int,default=SAMPLES) - parser.add_argument('--keep_cols',type=str,default=COLS) - parser.add_argument('--hidden_layers',type=int,default=HIDDEN_LAYERS) - parser.add_argument('--latent_dim',type=int,default=LATENT_DIM) - parser.add_argument('--hidden_dim',type=int,nargs='+',default=HIDDEN_DIM) - parser.add_argument('--output_activation',type=str,default=OUTPUT_ACTIVATION) - parser.add_argument('--epochs',type=int,default=EPOCHS) - parser.add_argument('--batch_size',type=int,default=BATCH_SIZE) - parser.add_argument('--learning_rate',type=float,default=LEARNING_RATE) - parser.add_argument('--clip',type=float,nargs='+',default=CLIP) - parser.add_argument('--standardized', default=STANDARDIZED, action='store_true') - parser.add_argument('--minmax', default=MINMAX, action='store_true') - parser.add_argument('--print_progress', default=PRINT_PROGRESS, action='store_true') - parser.add_argument('--save', default=SAVE, action='store_true') - parser.add_argument('--save_path',type=str,default=SAVE_PATH) - parser.add_argument('--load_path',type=str,default=LOAD_PATH) - parser.add_argument('--model_name',type=str,default=MODEL_NAME) - parser.add_argument('--continue_training', default=CONTINUE_TRAINING, action='store_true') - args = parser.parse_args() - - run(args) \ No newline at end of file diff --git a/components/outlier-detection/vae/utils.py b/components/outlier-detection/vae/utils.py deleted file mode 100644 index 569dd54ba9..0000000000 --- a/components/outlier-detection/vae/utils.py +++ /dev/null @@ -1,171 +0,0 @@ -import collections -import json -import numpy as np -import pandas as pd -import requests -from sklearn.datasets import fetch_kddcup99 -from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score, fbeta_score - -pd.options.mode.chained_assignment = None # default='warn' - -def get_kdd_data(target=['dos','r2l','u2r','probe'], - keep_cols=['srv_count','serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate', - 'same_srv_rate','diff_srv_rate','srv_diff_host_rate','dst_host_count','dst_host_srv_count', - 'dst_host_same_srv_rate','dst_host_diff_srv_rate','dst_host_same_src_port_rate', - 'dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate','target'], - percent10=False): - """ Load KDD Cup 1999 data and return in dataframe. """ - - data_raw = fetch_kddcup99(subset=None, data_home=None, percent10=percent10) - - # specify columns - cols=['duration','protocol_type','service','flag','src_bytes','dst_bytes','land','wrong_fragment','urgent','hot', - 'num_failed_logins','logged_in','num_compromised','root_shell','su_attempted','num_root','num_file_creations', - 'num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','count','srv_count', - 'serror_rate','srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate','diff_srv_rate', - 'srv_diff_host_rate','dst_host_count','dst_host_srv_count','dst_host_same_srv_rate','dst_host_diff_srv_rate', - 'dst_host_same_src_port_rate','dst_host_srv_diff_host_rate','dst_host_serror_rate','dst_host_srv_serror_rate', - 'dst_host_rerror_rate','dst_host_srv_rerror_rate'] - - # create dataframe - data = pd.DataFrame(data=data_raw['data'],columns=cols) - - # add target to dataframe - data['attack_type'] = data_raw['target'] - - # specify and map attack types - attack_list = np.unique(data['attack_type']) - attack_category = ['dos','u2r','r2l','r2l','r2l','probe','dos','u2r','r2l','dos','probe','normal','u2r', - 'r2l','dos','probe','u2r','probe','dos','r2l','dos','r2l','r2l'] - - attack_types = {} - for i,j in zip(attack_list,attack_category): - attack_types[i] = j - - data['attack_category'] = 'normal' - for key,value in attack_types.items(): - data['attack_category'][data['attack_type'] == key] = value - - # define target - data['target'] = 0 - for t in target: - data['target'][data['attack_category'] == t] = 1 - - # define columns to be dropped - drop_cols = [] - for col in data.columns.values: - if col not in keep_cols: - drop_cols.append(col) - - if drop_cols!=[]: - data.drop(columns=drop_cols,inplace=True) - - return data - - -def sample_df(df,n): - """ Sample from df. """ - if n < df.shape[0]+1: - replace = False - else: - replace = True - return df.sample(n=n,replace=replace) - - -def generate_batch(data,n_samples,frac_outliers): - """ Generate random batch from data with fixed size and fraction of outliers. """ - - normal = data[data['target']==0] - outlier = data[data['target']==1] - - if n_samples==1: - n_outlier = np.random.binomial(1,frac_outliers) - n_normal = 1 - n_outlier - else: - n_normal = int((1-frac_outliers) * n_samples) - n_outlier = int(frac_outliers * n_samples) - - batch_normal = sample_df(normal,n_normal) - batch_outlier = sample_df(outlier,n_outlier) - - batch = pd.concat([batch_normal,batch_outlier]) - batch = batch.sample(frac=1).reset_index(drop=True) - - outlier_true = batch['target'].values - batch.drop(columns=['target'],inplace=True) - - return batch.values.astype('float'), outlier_true - -def flatten(x): - if isinstance(x, collections.Iterable): - return [a for i in x for a in flatten(i)] - else: - return [x] - -def performance(y_true,y_pred,roll_window=100): - """ Return a confusion matrix and calculate rolling accuracy, precision, recall, F1 and F2 scores. """ - - # confusion matrix - cm = confusion_matrix(y_true,y_pred,labels=[0,1]) - tn, fp, fn, tp = cm.ravel() - - # total scores - acc_tot = accuracy_score(y_true,y_pred) - prec_tot = precision_score(y_true,y_pred) - rec_tot = recall_score(y_true,y_pred) - f1_tot = f1_score(y_true,y_pred) - f2_tot = fbeta_score(y_true,y_pred,beta=2) - - # rolling scores - y_true_roll = y_true[-roll_window:] - y_pred_roll = y_pred[-roll_window:] - acc_roll = accuracy_score(y_true_roll,y_pred_roll) - prec_roll = precision_score(y_true_roll,y_pred_roll) - rec_roll = recall_score(y_true_roll,y_pred_roll) - f1_roll = f1_score(y_true_roll,y_pred_roll) - f2_roll = fbeta_score(y_true_roll,y_pred_roll,beta=2) - - scores = [tn, fp, fn, tp, acc_tot, prec_tot, rec_tot, f1_tot, f2_tot, - acc_roll, prec_roll, rec_roll, f1_roll, f2_roll] - - return scores - -def outlier_stats(y_true,y_pred,roll_window=100): - """ Calculate number and percentage of predicted and labeled outliers. """ - - y_pred_roll = np.sum(y_pred[-roll_window:]) - y_true_roll = np.sum(y_true[-roll_window:]) - y_pred_tot = np.sum(y_pred) - y_true_tot = np.sum(y_true) - - return y_pred_roll, y_true_roll, y_pred_tot, y_true_tot - -def get_payload(arr): - features = ["srv_count","serror_rate","srv_serror_rate","rerror_rate","srv_rerror_rate","same_srv_rate", - "diff_srv_rate","srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate", - "dst_host_diff_srv_rate","dst_host_same_src_port_rate","dst_host_srv_diff_host_rate", - "dst_host_serror_rate","dst_host_srv_serror_rate","dst_host_rerror_rate","dst_host_srv_rerror_rate"] - datadef = {"names":features,"ndarray":arr.tolist()} - payload = {"meta":{},"data":datadef} - return payload - -def rest_request_ambassador(deploymentName,namespace,request,endpoint="localhost:8003"): - response = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/predictions", - json=request) - print(response.status_code) - print(response.text) - return response.json() - -def send_feedback_rest(deploymentName,namespace,request,response,reward,truth,endpoint="localhost:8003"): - feedback = { - "request": request, - "response": response, - "reward": reward, - "truth": {"data":{"ndarray":truth.tolist()}} - } - ret = requests.post( - "http://"+endpoint+"/seldon/"+namespace+"/"+deploymentName+"/api/v0.1/feedback", - json=feedback) - return