Alarms

overwatch/processing/alarms/README.md

@@ -0,0 +1,164 @@
+
+# Alarms
+
+This module is responsible for generating alarms and sending notifications about them.
+
+Class Alarm has an abstract method `checkAlarm()`, which allows us to implement our own alarms.
+
+Alarms can be aggregated by logic functions or/and.
+
+Examples of alarms can be found in impl package.
+
+## BetweenValuesAlarm
+
+Checks if trend is between minimal and maximal allowed values.
+
+![betweenAlarm example](./doc/betweenAlarm.png)
+
+## MeanInRangeAlarm
+
+Checks if mean from N last measurements is in the range.
+![No alarm](./doc/meanRange1.png)
+![No alarm](./doc/meanRange2.png)
+![No alarm](./doc/meanRange3.png)
+![Alarm](./doc/meanRange4.png)
+
+## AbsolutePreviousValueAlarm
+
+Check if (new value - old value) is different more than delta.
+
+![absolutePreviousValueAlarm example](./doc/absolute.png)
+
+## RelativeProviousValueAlarm
+
+Check if new value is between (previous value)/ratio and (previous value)*ratio.
+
+![No alarm](./doc/relative.png)
+![Alarm](./doc/relative1.png)
+![No alarm](./doc/relative2.png)
+![Alarm](./doc/relative3.png)
+
+
+## CheckLastNAlarm
+
+Check if minimum ratio*N last N alarms are in range.
+
+![No alarm](./doc/last.png)
+![No alarm](./doc/last1.png)
+![Alarm](./doc/last2.png)
+![No alarm](./doc/last3.png)
+
+
+## Displaying on the webApp
+When histogram is processed and alarms are generated, they are displayed above this histogram on the webApp.
+
+# Class Diagram
+![Diagram](./doc/alarms_class_diag.png)
+
+# Notifications
+
+Each generated alarm is collected by AlarmCollector. It allows us send notifications about alarms when we want:
+after processing trending object, after processing histogram or when all histograms are processed. To send messages via email
+you have to call
+`announceOnEmail()` method on alarmCollector object. To print messages on console call `showOnConsole()` method. To
+send on Slack call `announceOnSlack()`
+
+## Emails
+
+There is possibility to send notifications about alarms via email. To send emails add to configuration file following information:
+
+```yaml
+# Email configuration for gmail
+emailDelivery:
+    smtpSettings:
+      address: "smtp.gmail.com"
+      port: 587
+      userName: "email@address"
+      password: "password"
+```
+
+## Slack
+
+To send messages on Slack add to configuration file:
+
+```yaml
+# Slack configuration
+# Define token and channel
+slack:
+  apiToken: 'token'
+  slackChannel: "alarms"
+```
+
+# Usage
+
+To specify alarms and receivers write for example following function:
+
+```python
+def alarmConfig():
+    recipients = ["test1@mail", "test2@mail"]
+    mailSender = MailSender(recipients)
+    slackSender = SlackNotification()
+    borderWarning = BetweenValuesAlarm(minVal=0, maxVal=50, alarmText="WARNING")
+
+    borderWarning.receivers = [printCollector]
+
+    borderError = BetweenValuesAlarm(minVal=0, maxVal=70, alarmText="ERROR")
+    borderError.receivers = [mailSender, slackSender]
+
+    bva = BetweenValuesAlarm(minVal=0, maxVal=90, alarmText='BETWEEN')
+    # TODO add second alarm to andAlarm
+    seriousAlarm = AndAlarm([bva], "Serious Alarm")
+    seriousAlarm.addReceiver(mailSender)
+
+    return [borderWarning, borderError, seriousAlarm]
+```
+
+You can define separate alarms to different trending objects:
+
+```python
+def alarmMeanConfig():
+    slack = SlackNotification()
+    lastAlarm = CheckLastNAlarm(alarmText="ERROR")
+    lastAlarm.receivers = [printCollector, slack]
+
+    return [lastAlarm]
+
+def alarmStdConfig():
+    slack = SlackNotification()
+    meanInRangeWarning = MeanInRangeAlarm(alarmText="WARNING")
+    meanInRangeWarning.receivers = [printCollector, slack]
+
+    return [meanInRangeWarning]
+
+def alarmMaxConfig():
+    recipients = ["test1@mail", "test2@mail"]
+    mailSender = MailSender(recipients)
+    borderWarning = BetweenValuesAlarm(minVal=0, maxVal=50, alarmText="WARNING")
+    borderWarning.receivers = [printCollector, mailSender]
+
+    return [borderWarning]
+```
+
+To use alarms, define them in EMC.py or other detector file in `getTrendingObjectInfo()`:
+
+```python
+     trendingNameToObject = {
+        "max": trendingObjects.MaximumTrending,
+        "mean": trendingObjects.MeanTrending,
+        "stdDev": trendingObjects.StdDevTrending,
+    }
+    # Assign created earlier alarms to particular trending objects
+    alarms = {
+        "max": alarmMaxConfig(),
+        "mean": alarmMeanConfig(),
+        "stdDev": alarmStdConfig()
+    }
+    trendingInfo = []
+    for prefix, cls in trendingNameToObject.items():
+        for dependingFile, desc in infoList:
+            infoObject = TrendingInfo(prefix + dependingFile, desc, [dependingFile], cls)
+            if prefix in alarms:
+                infoObject.addAlarm(alarms[prefix])       # Set alarms
+            trendingInfo.append(infoObject)
+    return trendingInfo
+```

overwatch/processing/alarms/aggregatingAlarm.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+""" Base class for alarms which manage of aggregation alarms.
+
+.. code-author: Pawel Ostrowski <[email protected]>, AGH University of Science and Technology
+"""
+
+from overwatch.processing.alarms.alarm import Alarm
+
+try:
+    from typing import *  # noqa
+except ImportError:
+    pass
+
+
+class AggregatingAlarm(Alarm):
+    def __init__(self, children, alarmText=''):  # type: (List[Alarm], str) -> None
+        super(AggregatingAlarm, self).__init__(alarmText=alarmText)
+
+        # None - no value, True/False - last value returned from alarm
+        self.children = {c: None for c in children}  # type: Dict[Alarm, Optional[bool]]
+
+        for child in children:
+            child.parent = self
+
+    def addChild(self, child):  # type: (Alarm) -> None
+        assert child.parent is None
+        child.parent = self
+        self.children[child] = None
+
+    def isAllAlarmsCompleted(self):
+        return all(c is not None for c in self.children.values())
+
+    def checkAlarm(self, *args, **kwargs):  # type: () -> (bool, str)
+        """abstract method"""
+        raise NotImplementedError
+
+    def childProcessed(self, child, result):
+        if self.children[child] is not None:
+            print("WARNING: last result ignored")
+
+        self.children[child] = result
+
+        if self.isAllAlarmsCompleted():
+            self.processCheck()
+            self.children = {c: None for c in self.children}

overwatch/processing/alarms/alarm.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+""" Base class for alarms.
+
+.. code-author: Pawel Ostrowski <[email protected]>, AGH University of Science and Technology
+"""
+import numpy as np
+from overwatch.processing.alarms.collectors import alarmCollector
+
+try:
+    from typing import *  # noqa
+except ImportError:
+    # Imports in this block below here are used solely for typing information
+    from ..trending.objects.object import TrendingObject  # noqa
+    from overwatch.processing.alarms.aggregatingAlarm import AggregatingAlarm  # noqa
+
+
+class Alarm(object):
+    def __init__(self, alarmText='', collector=None):
+        self.alarmText = alarmText
+        self.collector = collector
+        self.receivers = []
+        self.parent = None  # type: Optional[AggregatingAlarm]
+
+    def addReceiver(self, receiver):  # type: (callable) -> None
+        self.receivers.append(receiver)
+
+    def processCheck(self, trend=None):  # type: (Optional[TrendingObject]) -> None
+        args = (self.prepareTrendValues(trend),) if trend else ()
+        result = self.checkAlarm(*args)
+        isAlarm, msg = result
+
+        if isAlarm:
+            msg = "[{alarmText}]: {msg}".format(alarmText=self.alarmText, msg=msg)
+
+            # aggregating alarms don't have trend
+            if trend:
+                trend.alarmsMessages.append(msg)
+
+            # tell collector about alarm or announce alarm itself
+            if self.collector:
+                alarmCollector.collectMessage(self, "[{trendName}]{msg}".format(trendName=trend.name, msg=msg))
+            else:
+                self._announceAlarm(msg)
+        if self.parent:
+            self.parent.childProcessed(child=self, result=isAlarm)
+
+    @staticmethod
+    def prepareTrendValues(trend):  # type: (TrendingObject) -> np.ndarray
+        trendingValues = np.array(trend.trendedValues)
+        if len(trendingValues.shape) == 2:
+            trendingValues = trendingValues[:, 0]
+        if len(trendingValues.shape) > 2:
+            raise TypeError
+        return trendingValues
+
+    def checkAlarm(self, trend):  # type: (np.ndarray) -> (bool, str)
+        """abstract method"""
+        raise NotImplementedError
+
+    def _announceAlarm(self, msg):  # type: (str) -> None
+        for receiver in self.receivers:
+            receiver(msg)