[filebeat][threatintel] Ignore bad indicator IPs for MISP fileset (#3…

…4195)

MISP may send an Event.Attribute.value IP as a CIDR such as
146.88.240.0/24, which is not a valid IP per the Elasticsearch
IP data type.

Co-authored-by: Dan Kortschak <[email protected]>

CHANGELOG.next.asciidoc

@@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 *Filebeat*
 
+- Allow the `misp` fileset in the Filebeat `threatintel` module to ignore CIDR ranges for an IP field. {issue}29949[29949] {pull}34195[34195]
 
 *Auditbeat*
 

x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml

@@ -249,6 +249,8 @@ processors:
         - "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}"
       ignore_missing: true
       if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
+      # MISP may send a CIDR such as 1.2.3.0/22, which is not a valid Elasticsearch IP data type
+      ignore_failure: true
 
   ## Email indicator operations
   # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp.

x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log

@@ -0,0 +1 @@
+{"Event":{"id":"10","orgc_id":"4","org_id":"1","date":"2020-12-09","threat_level_id":"3","info":"Recent Qakbot (Qbot) activity","published":true,"uuid":"5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16","attribute_count":"15","analysis":"2","timestamp":"1607868196","distribution":"3","proposal_email_lock":false,"locked":false,"publish_timestamp":"1610637888","sharing_group_id":"0","disable_correlation":false,"extends_uuid":"","Org":{"id":"1","name":"ORGNAME","uuid":"5877549f-ea76-4b91-91fb-c72ad682b4a5","local":true},"Orgc":{"id":"4","name":"CUDESO","uuid":"56c42374-fdb8-4544-a218-41ffc0a8ab16","local":false},"Attribute":{"id":"10686","type":"ip-dst|port","category":"Network activity","to_ids":true,"uuid":"5fd0c620-a844-4ace-9710-a37bc0a8ab16","event_id":"10","distribution":"5","timestamp":"1607517728","comment":"On port 2222","sharing_group_id":"0","deleted":false,"disable_correlation":false,"object_id":"0","object_relation":null,"first_seen":null,"last_seen":null,"value":"146.88.240.0/24","Galaxy":[],"ShadowAttribute":[]},"ShadowAttribute":[],"RelatedEvent":[],"Galaxy":[],"Object":[],"EventReport":[],"Tag":[{"id":"3","name":"tlp:white","colour":"#ffffff","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":false,"is_custom_galaxy":false,"local":0},{"id":"6","name":"misp-galaxy:banker=\"Qakbot\"","colour":"#0088cc","exportable":true,"user_id":"0","hide_tag":false,"numerical_value":null,"is_galaxy":true,"is_custom_galaxy":false,"local":0}]}}

x-pack/filebeat/module/threatintel/misp/test/misp_sample_cidr.ndjson.log-expected.json

@@ -0,0 +1,60 @@
+[
+    {
+        "@timestamp": "2020-12-13T14:03:16.000Z",
+        "event.category": "threat",
+        "event.dataset": "threatintel.misp",
+        "event.kind": "enrichment",
+        "event.module": "threatintel",
+        "event.timezone": "-02:00",
+        "event.type": "indicator",
+        "fileset.name": "misp",
+        "input.type": "log",
+        "log.offset": 0,
+        "misp.attribute.category": "Network activity",
+        "misp.attribute.comment": "On port 2222",
+        "misp.attribute.deleted": false,
+        "misp.attribute.disable_correlation": false,
+        "misp.attribute.distribution": 5,
+        "misp.attribute.event_id": "10",
+        "misp.attribute.id": "10686",
+        "misp.attribute.object_id": "0",
+        "misp.attribute.sharing_group_id": "0",
+        "misp.attribute.timestamp": "1607517728",
+        "misp.attribute.to_ids": true,
+        "misp.attribute.type": "ip-dst|port",
+        "misp.attribute.uuid": "5fd0c620-a844-4ace-9710-a37bc0a8ab16",
+        "misp.attribute_count": 15,
+        "misp.date": "2020-12-09",
+        "misp.disable_correlation": false,
+        "misp.distribution": "3",
+        "misp.extends_uuid": "",
+        "misp.id": "10",
+        "misp.info": "Recent Qakbot (Qbot) activity",
+        "misp.locked": false,
+        "misp.org_id": "1",
+        "misp.orgc.id": "4",
+        "misp.orgc.local": false,
+        "misp.orgc.name": "CUDESO",
+        "misp.orgc.uuid": "56c42374-fdb8-4544-a218-41ffc0a8ab16",
+        "misp.orgc_id": "4",
+        "misp.proposal_email_lock": false,
+        "misp.publish_timestamp": "1610637888",
+        "misp.published": true,
+        "misp.sharing_group_id": "0",
+        "misp.threat_level_id": 3,
+        "misp.uuid": "5fd0c599-ab6c-4ba1-a69a-df9ec0a8ab16",
+        "service.type": "threatintel",
+        "tags": [
+            "misp-galaxy:banker=Qakbot",
+            "tlp:white"
+        ],
+        "threat.feed.dashboard_id": "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f",
+        "threat.feed.name": "[Filebeat] MISP",
+        "threat.indicator.marking.tlp": [
+            "white"
+        ],
+        "threat.indicator.provider": "misp",
+        "threat.indicator.scanner_stats": 2,
+        "threat.indicator.type": "ipv4-addr"
+    }
+]