Skip to content

Commit

Permalink
[ML] Fixes raw data drilldowns for Apache, Nginx, Auditbeat modules (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
peteharverson authored Jul 27, 2020
1 parent 25856ed commit d9a6461
Show file tree
Hide file tree
Showing 14 changed files with 40 additions and 66 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"groups": ["apache"],
"description": "HTTP Access Logs: Detect low request rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "doc_count",
"detectors": [
Expand All @@ -27,7 +27,7 @@
"custom_urls": [
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
{
"groups": ["apache"],
"description": "HTTP Access Logs: Detect unusual source IPs - high request rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
{
"detector_description": "Apache access source IP high count",
"function": "high_count",
"over_field_name": "source.address"
}
],
"influencers": [
"source.address"
]
"influencers": ["source.address"]
},
"data_description": {
"time_field": "@timestamp",
Expand All @@ -27,7 +25,7 @@
},
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,17 @@
{
"groups": ["apache"],
"description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
{
"detector_description": "Apache access source IP high dc URL",
"function": "high_distinct_count",
"field_name": "url.original",
"over_field_name": "source.address"
}
],
"influencers": [
"source.address"
]
"influencers": ["source.address"]
},
"data_description": {
"time_field": "@timestamp",
Expand All @@ -28,7 +26,7 @@
},
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
{
"groups": ["apache"],
"description": "HTTP Access Logs: Detect unusual status code rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
{
"detector_description": "Apache access status code rate",
"function": "count",
"partition_field_name": "http.response.status_code"
}
],
"influencers": [
"http.response.status_code",
"source.address"
]
"influencers": ["http.response.status_code", "source.address"]
},
"analysis_limits": {
"model_memory_limit": "100mb"
Expand All @@ -34,7 +31,7 @@
},
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"groups": ["apache"],
"description": "HTTP Access Logs: Detect unusual visitor rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "dc_source_address",
"detectors": [
Expand All @@ -27,7 +27,7 @@
"custom_urls": [
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(event.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,7 @@
"partition_field_name": "container.name"
}
],
"influencers": [
"container.name",
"process.executable"
]
"influencers": ["container.name", "process.executable"]
},
"analysis_limits": {
"model_memory_limit": "256mb",
Expand All @@ -35,7 +32,7 @@
{
"url_name": "Raw data",
"time_range": "1h",
"url_value": "discover#/ml_auditbeat_docker_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027container.name:\u0022$container.name$\u0022\u0027))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027container.name:\u0022$container.name$\u0022\u0027))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,7 @@
"partition_field_name": "container.name"
}
],
"influencers": [
"container.name",
"process.executable"
]
"influencers": ["container.name", "process.executable"]
},
"analysis_limits": {
"model_memory_limit": "256mb"
Expand All @@ -35,7 +32,7 @@
{
"url_name": "Raw data",
"time_range": "1h",
"url_value": "discover#/ml_auditbeat_docker_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027container.name:\u0022$container.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027container.name:\u0022$container.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,7 @@
"partition_field_name": "host.name"
}
],
"influencers": [
"host.name",
"process.executable"
]
"influencers": ["host.name", "process.executable"]
},
"analysis_limits": {
"model_memory_limit": "256mb"
Expand All @@ -34,7 +31,7 @@
{
"url_name": "Raw data",
"time_range": "1h",
"url_value": "discover#/ml_auditbeat_hosts_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,7 @@
"partition_field_name": "host.name"
}
],
"influencers": [
"host.name",
"process.executable"
]
"influencers": ["host.name", "process.executable"]
},
"analysis_limits": {
"model_memory_limit": "256mb"
Expand All @@ -35,7 +32,7 @@
{
"url_name": "Raw data",
"time_range": "1h",
"url_value": "discover#/ml_auditbeat_hosts_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"groups": ["nginx"],
"description": "HTTP Access Logs: Detect low request rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "doc_count",
"detectors": [
Expand All @@ -27,7 +27,7 @@
"custom_urls": [
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
{
"groups": ["nginx"],
"description": "HTTP Access Logs: Detect unusual source IPs - high request rates (ECS)",
"analysis_config" : {
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
{
"detector_description": "Nginx access source IP high count",
"function": "high_count",
"over_field_name": "source.address"
}
],
"influencers": [
"source.address"
]
"influencers": ["source.address"]
},
"data_description": {
"time_field": "@timestamp",
Expand All @@ -27,7 +25,7 @@
},
{
"url_name": "Raw data",
"url_value": "discover#/ml_http_access_filebeat_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
"url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:event.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(event.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
Expand Down
Loading

0 comments on commit d9a6461

Please sign in to comment.