Merge branch 'master' into opsgenie_source_entity

CHANGELOG.md

@@ -16,7 +16,10 @@
 ## New features
 - Add support for generating Kibana Discover URLs to Rocket.Chat alerter - [#260](https://github.com/jertel/elastalert2/pull/260) - @nsanorururu
 - Provide rule key/values as possible Jinja data inputs - [#281](https://github.com/jertel/elastalert2/pull/281) - @mrfroggg
+- Add Jinja support to alert_subject - [#318](https://github.com/jertel/elastalert2/pull/318) - @mrfroggg
 - Add securityContext and podSecurityContext to Helm chart - [#289](https://github.com/jertel/elastalert2/pull/289) - @lepouletsuisse
+- Favor match keys over colliding rule keys when resolving Jinja vars; also add alert_text_jinja unit test - [#311](https://github.com/jertel/elastalert2/pull/311) - @mrfroggg
+- Add support for `servicenow_impact` and `servicenow_urgency` parameters for ServiceNow alerter - [#316](https://github.com/jertel/elastalert2/pull/316) - @randolph-esnet
 
 ## Other changes
 - Continue fix for prometheus wrapper writeback function signature - [#256](https://github.com/jertel/elastalert2/pull/256) - @greut

docs/source/ruletypes.rst

@@ -1435,7 +1435,7 @@ Similarly to ``alert_subject``, ``alert_text`` can be further formatted using Ji
 
 1. Jinja Template
 
-By setting ``alert_text_type: alert_text_jinja`` you can use jinja2 templates in ``alert_text``. ::
+By setting ``alert_text_type: alert_text_jinja`` you can use jinja2 templates in ``alert_text`` and ``alert_subject``. ::
 
     alert_text_type: alert_text_jinja
 
@@ -2570,6 +2570,10 @@ Optional:
 
 ``servicenow_proxy``: By default ElastAlert will not use a network proxy to send notifications to ServiceNow. Set this option using ``hostname:port`` if you need to use a proxy. only supports https.
 
+``servicenow_impact``: An integer 1, 2, or 3 representing high, medium, and low respectively. This measures the effect of an incident on business processes.
+
+``servicenow_urgency``: An integer 1, 2, or 3 representing high, medium, and low respecitvely. This measures how long this incident can be delayed until there is a significant business impact.
+
 Example usage::
 
     alert:
@@ -2584,6 +2588,8 @@ Example usage::
     subcategory: "xxxxxx"
     cmdb_ci: "xxxxxx"
     caller_id: "xxxxxx"
+    servicenow_impact: 1
+    servicenow_urgenc: 3
 
 Slack
 ~~~~~

elastalert/alerters/servicenow.py

@@ -26,6 +26,8 @@ def __init__(self, rule):
         super(ServiceNowAlerter, self).__init__(rule)
         self.servicenow_rest_url = self.rule.get('servicenow_rest_url', None)
         self.servicenow_proxy = self.rule.get('servicenow_proxy', None)
+        self.impact = self.rule.get('servicenow_impact', None)
+        self.urgency = self.rule.get('servicenow_urgency', None)
 
     def alert(self, matches):
         for match in matches:
@@ -48,6 +50,10 @@ def alert(self, matches):
             "cmdb_ci": self.rule['cmdb_ci'],
             "caller_id": self.rule["caller_id"]
         }
+        if self.impact != None:
+            payload["impact"] = self.impact
+        if self.urgency != None:
+            payload["urgency"] = self.urgency
         try:
             response = requests.post(
                 self.servicenow_rest_url,

elastalert/alerters/thehive.py

@@ -34,7 +34,7 @@ def load_observable_artifacts(self, match: dict):
         artifacts = []
         for mapping in self.rule.get('hive_observable_data_mapping', []):
             for observable_type, mapping_key in mapping.items():
-                data = self.lookup_field(match, mapping_key, '')
+                data = str(self.lookup_field(match, mapping_key, ''))
                 if len(data) != 0:
                     artifact = {'tlp': 2,
                                 'tags': [],

elastalert/alerts.py

@@ -3,6 +3,7 @@
 import json
 import os
 
+from jinja2 import Template
 from texttable import Texttable
 
 from elastalert.util import EAException, lookup_es_key
@@ -35,9 +36,9 @@ def _add_custom_alert_text(self):
             #  Top fields are accessible via `{{field_name}}` or `{{jinja_root_name['field_name']}}`
             #  `jinja_root_name` dict is useful when accessing *fields with dots in their keys*,
             #  as Jinja treat dot as a nested field.
-            alert_text = self.rule.get("jinja_template").render(**self.match, **self.rule,
-                                                                **{self.rule['jinja_root_name']: {**self.match,
-                                                                                                  **self.rule}})
+            template_values = self.rule | self.match
+            alert_text = self.rule.get("jinja_template").render(
+                template_values | {self.rule['jinja_root_name']: template_values})
         elif 'alert_text_args' in self.rule:
             alert_text_args = self.rule.get('alert_text_args')
             alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args]
@@ -210,7 +211,10 @@ def create_custom_title(self, matches):
             missing = self.rule.get('alert_missing_value', '<MISSING VALUE>')
             alert_subject_values = [missing if val is None else val for val in alert_subject_values]
             alert_subject = alert_subject.format(*alert_subject_values)
-
+        elif self.rule.get('alert_text_type') == "alert_text_jinja":
+            title_template = Template(str(self.rule.get('alert_subject', '')))
+            template_values = self.rule | matches[0]
+            alert_subject = title_template.render(template_values | {self.rule['jinja_root_name']: template_values})
         if len(alert_subject) > alert_subject_max_len:
             alert_subject = alert_subject[:alert_subject_max_len]
 

elastalert/schema.yaml

@@ -508,6 +508,8 @@ properties:
   cmdb_ci: {type: string}
   caller_id: {type: string}
   servicenow_proxy: {type: string}
+  servicenow_impact: {type: integer, minimum: 1, maximum: 3}
+  servicenow_urgency: {type: integer, minimum: 1, maximum: 3}
 
   ### Slack
   slack_webhook_url: *arrayOfString

requirements-dev.txt

@@ -5,7 +5,7 @@ flake8-absolute-import
 m2r2
 pluggy>=0.12.0
 pre-commit
-pylint<2.9
+pylint<2.10
 pytest==6.2.4
 pytest-cov==2.12.1
 pytest-xdist==2.3.0

requirements.txt

@@ -7,7 +7,7 @@ croniter>=0.3.16
 elasticsearch==7.0.0
 envparse>=0.2.0
 exotel>=0.1.3
-Jinja2==2.11.3
+Jinja2==3.0.1
 jira>=2.0.0
 jsonschema>=3.0.2
 prison>=0.1.2

tests/alerters/servicenow_test.py

@@ -117,6 +117,39 @@ def test_service_now_proxy():
     assert expected_data == actual_data
 
 
+def test_service_now_impact_and_urgency():
+    rule = {
+        'name': 'Test ServiceNow Rule',
+        'type': 'any',
+        'username': 'ServiceNow username',
+        'password': 'ServiceNow password',
+        'servicenow_rest_url': 'https://xxxxxxxxxx',
+        'short_description': 'ServiceNow short_description',
+        'comments': 'ServiceNow comments',
+        'assignment_group': 'ServiceNow assignment_group',
+        'category': 'ServiceNow category',
+        'subcategory': 'ServiceNow subcategory',
+        'cmdb_ci': 'ServiceNow cmdb_ci',
+        'caller_id': 'ServiceNow caller_id',
+        'servicenow_impact': 3,
+        'servicenow_urgency': 1,
+        'alert': []
+    }
+    rules_loader = FileRulesLoader({})
+    rules_loader.load_modules(rule)
+    alert = ServiceNowAlerter(rule)
+    match = {
+        '@timestamp': '2021-01-01T00:00:00',
+        'somefield': 'foobarbaz'
+    }
+    with mock.patch('requests.post') as mock_post_request:
+        alert.alert([match])
+
+    data = json.loads(mock_post_request.call_args_list[0][1]['data'])
+    assert data['impact'] == rule['servicenow_impact']
+    assert data['urgency'] == rule['servicenow_urgency']
+
+
 def test_service_now_ea_exception():
     with pytest.raises(EAException) as ea:
         rule = {

tests/alerters/thehive_test.py

@@ -29,7 +29,7 @@ def test_thehive_alerter(caplog):
             'hive_connection': {'hive_apikey': '',
                                 'hive_host': 'https://localhost',
                                 'hive_port': 9000},
-            'hive_observable_data_mapping': [{'ip': 'test.ip'}],
+            'hive_observable_data_mapping': [{'ip': 'test.ip', 'autonomous-system': 'test.as_number'}],
             'name': 'test-thehive',
             'tags': ['a', 'b'],
             'type': 'any'}
@@ -39,7 +39,8 @@ def test_thehive_alerter(caplog):
     match = {
         "test": {
           "ip": "127.0.0.1",
-          "port": 9876
+          "port": 9876,
+          "as_number": 1234
         },
         "@timestamp": "2021-05-09T14:43:30",
     }
@@ -54,6 +55,13 @@ def test_thehive_alerter(caplog):
                 "message": None,
                 "tags": [],
                 "tlp": 2
+            },
+            {
+                "data": "1234",
+                "dataType": "autonomous-system",
+                "message": None,
+                "tags": [],
+                "tlp": 2
             }
         ],
         "customFields": {

tests/alerts_test.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 import datetime
 import json
-
 from unittest import mock
+from jinja2 import Template
 
 from elastalert.alerts import Alerter
 from elastalert.alerts import BasicMatchString
@@ -157,6 +157,32 @@ def test_alert_text_kw_global_substitution(ea):
     assert 'Abc: abc from match' in alert_text
 
 
+def test_alert_text_jinja(ea):
+    rule = ea.rules[0].copy()
+    rule['foo_rule'] = 'foo from rule'
+    rule['owner'] = 'the owner from rule'
+    rule['abc'] = 'abc from rule'
+    rule['alert_text'] = 'Owner: {{owner}}; Foo: {{_data["foo_rule"]}}; Abc: {{abc}}; Xyz: {{_data["xyz"]}}'
+    rule['alert_text_type'] = "alert_text_jinja"
+    rule['jinja_root_name'] = "_data"
+    rule['jinja_template'] = Template(str(rule['alert_text']))
+
+    match = {
+        '@timestamp': '2016-01-01',
+        'field': 'field_value',
+        'abc': 'abc from match',
+        'xyz': 'from match'
+    }
+
+    alert_text = str(BasicMatchString(rule, match))
+    assert 'Owner: the owner from rule' in alert_text
+    assert 'Foo: foo from rule' in alert_text
+    assert 'Xyz: from match' in alert_text
+
+    # When the key exists in both places, it will come from the match
+    assert 'Abc: abc from match' in alert_text
+
+
 def test_resolving_rule_references():
     rule = {
         'name': 'test_rule',
@@ -248,3 +274,25 @@ def test_alert_subject_size_limit_with_args(ea):
     alert = Alerter(rule)
     alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}])
     assert 6 == len(alertSubject)
+
+
+def test_alert_subject_with_jinja():
+    rule = {
+        'name': 'test_rule',
+        'type': mock_rule(),
+        'owner': 'the_owner',
+        'priority': 2,
+        'alert_subject': 'Test alert for {{owner}}; field {{field}}; Abc: {{_data["abc"]}}',
+        'alert_text_type': "alert_text_jinja",
+        'jinja_root_name': "_data"
+    }
+    match = {
+        '@timestamp': '2016-01-01',
+        'field': 'field_value',
+        'abc': 'abc from match',
+    }
+    alert = Alerter(rule)
+    alertsubject = alert.create_custom_title([match])
+    assert "Test alert for the_owner;" in alertsubject
+    assert "field field_value;" in alertsubject
+    assert "Abc: abc from match" in alertsubject