From f47347b7480ea0d0876bda22ba434f74e0a06c63 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Tue, 13 Jun 2023 08:50:22 +0200 Subject: [PATCH] Fix url scheme grok pattern --- CHANGELOG-developer.next.asciidoc | 1 + .../module/panw/panos/ingest/pipeline.yml | 2 +- .../module/panw/panos/test/pan_inc_threat.log | 2 +- .../test/pan_inc_threat.log-expected.json | 84 +++++++++---------- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 5ff66ad373bd..fe7fb5c57287 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -82,6 +82,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix the ingest pipeline for mysql slowlog to parse schema name with dash {pull}34371[34372] - Fix the multiple host support for mongodb module {pull}34624[34624] - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138] +- Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757] ==== Added diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 5d5941561ff1..7e34e24fdc2f 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -437,7 +437,7 @@ processors: - grok: field: url.original patterns: - - '(%{ANY:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?' + - '(%{URIPROTO:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?' ignore_missing: true pattern_definitions: USERNAME: '[^\:]*' diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log index fff6477c1e40..989fc3e8bd43 100644 --- a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log +++ b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log @@ -62,7 +62,7 @@ Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42: Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42:42,192.168.0.2,213.180.199.61,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:42:42,28932,1,59721,80,0,0,0x200000,tcp,block-url,"edw-melon.narod.ru/config.txt",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,Russian Federation,0, Mar 25 23:59:12 1,2013/03/25 23:59:12,01606001116,THREAT,url,1,2012/04/10 04:42:51,192.168.0.2,213.180.199.61,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:42:51,28953,1,59752,80,0,0,0x200000,tcp,block-url,"maximtushin.narod.ru/config.txt",(9999),malware-sites,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,Russian Federation,0, Mar 25 23:59:17 1,2013/03/25 23:59:17,01606001116,THREAT,file,1,2012/04/10 04:19:59,89.160.20.112,192.168.0.2,0.0.0.0,0.0.0.0,rule1,,crusher,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:20:05,64856,1,80,54431,0,0,0x200000,tcp,deny,"uLLGRaXP.exe",Windows Executable (EXE)(52020),any,low,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0, -Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,"marketingsoluchion.biz/fkn/config.bin",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0, +Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,"www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0, Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:27,192.168.0.6,207.46.140.46,0.0.0.0,0.0.0.0,rule1,jordy,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:32,25217,1,1047,80,0,0,0x200000,tcp,alert,"default.aspx",PII(60000),any,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0, Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:29,81.2.69.143,192.168.0.6,0.0.0.0,0.0.0.0,rule1,,jordy,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:34,25653,1,80,1039,0,0,0x200000,tcp,alert,"sck.aspx",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0, Mar 25 23:59:32 1,2013/03/25 23:59:32,01606001116,THREAT,data,1,2012/04/09 08:18:32,81.2.69.143,192.168.0.6,0.0.0.0,0.0.0.0,rule1,,jordy,web-browsing,vsys1,untrust,trust,ethernet1/2,ethernet1/1,forwardAll,2012/04/09 08:18:37,25717,3,80,1064,0,0,0x200000,tcp,alert,"ADSAdClient31.dll",PII(60000),any,informational,server-to-client,0,0x0,United States,192.168.0.0-192.168.255.255,0, diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json index 58ba2c93b7ce..9ec6337d6d0e 100644 --- a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json @@ -6010,7 +6010,7 @@ "event.dataset": "panw.panos", "event.kind": "alert", "event.module": "panw", - "event.original": "Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,\"marketingsoluchion.biz/fkn/config.bin\",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,", + "event.original": "Mar 25 23:59:22 1,2013/03/25 23:59:22,01606001116,THREAT,url,1,2012/04/10 04:09:01,192.168.0.2,67.43.156.12,0.0.0.0,0.0.0.0,rule1,crusher,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,forwardAll,2012/04/10 04:09:01,55402,1,63183,80,0,0,0x200000,tcp,block-url,\"www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/\",(9999),unknown,informational,client-to-server,0,0x0,192.168.0.0-192.168.255.255,United States,0,", "event.outcome": "success", "event.severity": 5, "event.timezone": "-02:00", @@ -6048,12 +6048,12 @@ "panw.panos.sub_type": "url", "panw.panos.threat.id": "9999", "panw.panos.threat.name": "URL-filtering", - "panw.panos.threat.resource": "marketingsoluchion.biz/fkn/config.bin", + "panw.panos.threat.resource": "www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "unknown", "panw.panos.virtual_sys": "vsys1", "related.hosts": [ - "marketingsoluchion.biz" + "www.sportspar.de" ], "related.ip": [ "0.0.0.0", @@ -6076,10 +6076,10 @@ "forwarded", "pan-os" ], - "url.domain": "marketingsoluchion.biz", - "url.extension": "bin", - "url.original": "marketingsoluchion.biz/fkn/config.bin", - "url.path": "/fkn/config.bin", + "url.domain": "www.sportspar.de", + "url.original": "www.sportspar.de/widgets/index/refreshStatistic?requestPage=/&requestController=index&referer=https://www.google.com/", + "url.path": "/widgets/index/refreshStatistic", + "url.query": "requestPage=/&requestController=index&referer=https://www.google.com/", "user.name": "crusher" }, { @@ -6113,7 +6113,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 26586, + "log.offset": 26666, "network.application": "web-browsing", "network.community_id": "1:KC3xpBK9CdouZqamG9S6Mjl6LIo=", "network.direction": "inbound", @@ -6198,7 +6198,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 26964, + "log.offset": 27044, "network.application": "web-browsing", "network.community_id": "1:oZUSrEMVr54enE9TsNjtdpJu0L8=", "network.direction": "outbound", @@ -6290,7 +6290,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 27336, + "log.offset": 27416, "network.application": "web-browsing", "network.community_id": "1:vpvx2rrEII2Wtti+NqSoe98K6s4=", "network.direction": "outbound", @@ -6382,7 +6382,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 27717, + "log.offset": 27797, "network.application": "web-browsing", "network.community_id": "1:MeB0cefg5kMN7f+LW+cirwH2nA8=", "network.direction": "inbound", @@ -6466,7 +6466,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 28086, + "log.offset": 28166, "network.application": "web-browsing", "network.community_id": "1:lI0hgoESF7/v82QAbsIMoPxInGQ=", "network.direction": "outbound", @@ -6560,7 +6560,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 28455, + "log.offset": 28535, "network.application": "pandora", "network.community_id": "1:c67I85z1uJV7VW6M9MR5Q8fjHQM=", "network.direction": "inbound", @@ -6644,7 +6644,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 28843, + "log.offset": 28923, "network.application": "google-maps", "network.community_id": "1:tsjbpnOPfE5+wHs/9MImDTjVjp8=", "network.direction": "outbound", @@ -6736,7 +6736,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "low", - "log.offset": 29215, + "log.offset": 29295, "network.application": "web-browsing", "network.community_id": "1:a/X3iTqQa+TxkHJgrAy4Npfe+ZM=", "network.direction": "outbound", @@ -6821,7 +6821,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 29590, + "log.offset": 29670, "network.application": "google-maps", "network.community_id": "1:Tc4KEUPBViPeku88f+PNN9tpeuc=", "network.direction": "outbound", @@ -6912,7 +6912,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 29962, + "log.offset": 30042, "network.application": "google-maps", "network.community_id": "1:OjvHxM13sIYbWzkV4RtvyxXDyVM=", "network.direction": "outbound", @@ -7004,7 +7004,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 30336, + "log.offset": 30416, "network.application": "google-maps", "network.community_id": "1:kYzGF0Llye+Lln7ejrGG5SI6mW8=", "network.direction": "outbound", @@ -7096,7 +7096,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 30710, + "log.offset": 30790, "network.application": "google-maps", "network.community_id": "1:AwfQlEV4j9qZjH7WG4q1qExon/o=", "network.direction": "outbound", @@ -7188,7 +7188,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 31082, + "log.offset": 31162, "network.application": "google-analytics", "network.community_id": "1:pRuFj5DzdmtFceU+OTawbYPhbJg=", "network.direction": "inbound", @@ -7272,7 +7272,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 31462, + "log.offset": 31542, "network.application": "google-maps", "network.community_id": "1:PFB0Gj5/utCZj8v3vJPCiBrGY3Y=", "network.direction": "outbound", @@ -7365,7 +7365,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 31836, + "log.offset": 31916, "network.application": "web-browsing", "network.community_id": "1:N/Bc1RgG30q1Owz0DWHR2yEwN44=", "network.direction": "outbound", @@ -7450,7 +7450,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 32215, + "log.offset": 32295, "network.application": "web-browsing", "network.community_id": "1:mSmmKo9krpIsh+2qFAZoA8nMDhg=", "network.direction": "outbound", @@ -7540,7 +7540,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 32600, + "log.offset": 32680, "network.application": "web-browsing", "network.community_id": "1:03rrdI/L+dbrLea/vrQULMTFqvU=", "network.direction": "outbound", @@ -7632,7 +7632,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 32974, + "log.offset": 33054, "network.application": "web-browsing", "network.community_id": "1:bJxw0tI76mNYOiv1ZJjBXdDpnTU=", "network.direction": "outbound", @@ -7721,7 +7721,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 33378, + "log.offset": 33458, "network.application": "google-maps", "network.community_id": "1:h4FhwHd9ztu4jpl3xgOaiB011a4=", "network.direction": "outbound", @@ -7812,7 +7812,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 33749, + "log.offset": 33829, "network.application": "google-maps", "network.community_id": "1:dULQBKOE61wtZ1QM6GKohdrM1GE=", "network.direction": "outbound", @@ -7904,7 +7904,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 34119, + "log.offset": 34199, "network.application": "rss", "network.community_id": "1:DLYH0WNYoXQ93i3rnp9QFsh63iM=", "network.direction": "outbound", @@ -7993,7 +7993,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 34486, + "log.offset": 34566, "network.application": "google-maps", "network.community_id": "1:jorKmgA/OY669gtX62Fasc1iKGc=", "network.direction": "outbound", @@ -8084,7 +8084,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 34858, + "log.offset": 34938, "network.application": "web-browsing", "network.community_id": "1:v/xhtv/qhJVgrOjMPvPqMWlrHXA=", "network.direction": "outbound", @@ -8170,7 +8170,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 35225, + "log.offset": 35305, "network.application": "web-browsing", "network.community_id": "1:lM6ErOc/Uj5ui7hk5LvnxpCB/K0=", "network.direction": "outbound", @@ -8261,7 +8261,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 35600, + "log.offset": 35680, "network.application": "google-maps", "network.community_id": "1:AFqpyz1JYwEsC+Bm2Q7fspI+r8Y=", "network.direction": "outbound", @@ -8363,7 +8363,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 35972, + "log.offset": 36052, "network.application": "google-analytics", "network.community_id": "1:8xEo6/LvOntD+xMHdXzKIXv9JxE=", "network.direction": "inbound", @@ -8447,7 +8447,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 36353, + "log.offset": 36433, "network.application": "google-maps", "network.community_id": "1:diAtdns9tWiH2bS++Pup9kMV+AI=", "network.direction": "outbound", @@ -8538,7 +8538,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 36725, + "log.offset": 36805, "network.application": "google-maps", "network.community_id": "1:cs7mutkQqIorGFAbWD2/09AnYXk=", "network.direction": "outbound", @@ -8630,7 +8630,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 37097, + "log.offset": 37177, "network.application": "pandora", "network.community_id": "1:PzMJQoALQDxnDaqwOEEz4zxyhHU=", "network.direction": "inbound", @@ -8714,7 +8714,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 37484, + "log.offset": 37564, "network.application": "google-maps", "network.community_id": "1:8xnlPG6iTh0CwnSMVwmWkniCAeM=", "network.direction": "outbound", @@ -8807,7 +8807,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 37857, + "log.offset": 37937, "network.application": "google-maps", "network.community_id": "1:SQGgi8ETBszNJv+EzlSRiGB/m5A=", "network.direction": "outbound", @@ -8900,7 +8900,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 38228, + "log.offset": 38308, "network.application": "google-maps", "network.community_id": "1:21uyYLV+/XbEeb+gCdBr5K1MWLU=", "network.direction": "outbound", @@ -8991,7 +8991,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 38597, + "log.offset": 38677, "network.application": "google-maps", "network.community_id": "1:QEEd+0of3hSmO6x9aRpIaHXdaUI=", "network.direction": "outbound", @@ -9083,7 +9083,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 38967, + "log.offset": 39047, "network.application": "google-analytics", "network.community_id": "1:BnyjuRL2HOxT/uRoNE3ra3neRSY=", "network.direction": "outbound", @@ -9174,7 +9174,7 @@ "input.type": "log", "labels.captive_portal": true, "log.level": "informational", - "log.offset": 39339, + "log.offset": 39419, "network.application": "google-maps", "network.community_id": "1:eGnclJrBulAHa+EiT+kLvValbJE=", "network.direction": "outbound",