Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parameterizing Painless script literals #9770

Merged
merged 11 commits into from
Dec 28, 2018
Prev Previous commit
Next Next commit
Reverting nginx change
  • Loading branch information
ycombinator committed Dec 27, 2018

Verified

This commit was signed with the committer’s verified signature.
pendo324 Justin
commit 2b141e639f1c6ff17ddca1fb02a8529af2b86b04
246 changes: 110 additions & 136 deletions filebeat/module/nginx/access/ingest/default.json
Original file line number Diff line number Diff line change
@@ -1,138 +1,112 @@
{
"description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:nginx.access.body_sent.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\""
],
"pattern_definitions": {
"IP_LIST": "%{IP}(\"?,?\\s*%{IP})*"
},
"ignore_missing": true
}
},
{
"grok": {
"field": "nginx.access.info",
"patterns": [
"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}",
""
],
"ignore_missing": true
}
},
{
"remove": {
"field": "nginx.access.info"
}
},
{
"split": {
"field": "network.forwarded_ip",
"separator": "\"?,?\\s+"
}
},
{
"set": {
"field": "source.ip",
"value": ""
}
},
{
"script": {
"source": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, params.dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.network.forwarded_ip) { if (!isPrivate(item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.network.forwarded_ip[0]; }",
"params": {
"dot": "."
}
}
},
{
"remove": {
"field": "message"
}
},
{
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
},
{
"date": {
"field": "nginx.access.time",
"target_field": "@timestamp",
"formats": [
"dd/MMM/YYYY:H:m:s Z"
]
}
},
{
"remove": {
"field": "nginx.access.time"
}
},
{
"user_agent": {
"field": "nginx.access.agent"
}
},
{
"rename": {
"field": "nginx.access.agent",
"target_field": "user_agent.original"
}
},
{
"rename": {
"field": "user_agent.os",
"target_field": "user_agent.os.full_name",
"ignore_missing": true
}
},
{
"rename": {
"field": "user_agent.os_name",
"target_field": "user_agent.os.name",
"ignore_missing": true
}
},
{
"rename": {
"field": "user_agent.os_major",
"target_field": "user_agent.os.major",
"ignore_missing": true
}
},
{
"rename": {
"field": "user_agent.os_minor",
"target_field": "user_agent.os.minor",
"ignore_missing": true
}
},
{
"rename": {
"field": "user_agent.os_patch",
"target_field": "user_agent.os.patch",
"ignore_missing": true
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
"description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:nginx.access.body_sent.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\""
],
"pattern_definitions": {
"IP_LIST": "%{IP}(\"?,?\\s*%{IP})*"
},
"ignore_missing": true
}
}, {
"grok": {
"field": "nginx.access.info",
"patterns": [
"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}",
""
],
"ignore_missing": true
}
}, {
"remove": {
"field": "nginx.access.info"
}
}, {
"split": {
"field": "network.forwarded_ip",
"separator": "\"?,?\\s+"
}
}, {
"set": {
"field": "source.ip",
"value": ""
}
}, {
"script": {
"lang": "painless",
"inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.network.forwarded_ip) { if (!isPrivate(item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.network.forwarded_ip[0]; }"
}
}, {
"remove":{
"field": "message"
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
}, {
"date": {
"field": "nginx.access.time",
"target_field": "@timestamp",
"formats": ["dd/MMM/YYYY:H:m:s Z"]
}
}, {
"remove": {
"field": "nginx.access.time"
}

}, { "user_agent": { "field": "nginx.access.agent" }
}, {
"rename": {
"field": "nginx.access.agent",
"target_field": "user_agent.original"
}
}, {
"rename": {
"field": "user_agent.os",
"target_field": "user_agent.os.full_name",
"ignore_missing": true
}
}, {
"rename": {
"field": "user_agent.os_name",
"target_field": "user_agent.os.name",
"ignore_missing": true
}
}, {
"rename": {
"field": "user_agent.os_major",
"target_field": "user_agent.os.major",
"ignore_missing": true
}
}, {
"rename": {
"field": "user_agent.os_minor",
"target_field": "user_agent.os.minor",
"ignore_missing": true
}
}, {
"rename": {
"field": "user_agent.os_patch",
"target_field": "user_agent.os.patch",
"ignore_missing": true
}

},{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
}],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}