From fc82d003264598254b828f08e8d2225ee72b57dc Mon Sep 17 00:00:00 2001 From: Pablo Sichert Date: Sat, 30 Jan 2021 23:46:14 +0100 Subject: [PATCH] Make field names of common log message more concise Signed-off-by: Pablo Sichert --- .../remap/functions/parse_common_log.cue | 40 +++--- lib/remap-functions/src/parse_common_log.rs | 123 ++++++------------ 2 files changed, 58 insertions(+), 105 deletions(-) diff --git a/docs/reference/remap/functions/parse_common_log.cue b/docs/reference/remap/functions/parse_common_log.cue index 0fe809d18aeec..f89ced2cb7dda 100644 --- a/docs/reference/remap/functions/parse_common_log.cue +++ b/docs/reference/remap/functions/parse_common_log.cue @@ -40,16 +40,16 @@ remap: functions: parse_common_log: { parse_common_log("127.0.0.1 bob frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326") """# return: { - remote_host: "127.0.0.1" - remote_logname: "bob" - auth_user: "frank" - timestamp: "2000-10-10T20:55:36Z" - request_line: "GET /apache_pb.gif HTTP/1.0" - request_method: "GET" - request_path: "/apache_pb.gif" - request_protocol: "HTTP/1.0" - status_code: 200 - content_length: 2326 + host: "127.0.0.1" + identity: "bob" + user: "frank" + timestamp: "2000-10-10T20:55:36Z" + message: "GET /apache_pb.gif HTTP/1.0" + method: "GET" + path: "/apache_pb.gif" + protocol: "HTTP/1.0" + status: 200 + size: 2326 } }, { @@ -61,16 +61,16 @@ remap: functions: parse_common_log: { ) """# return: { - remote_host: "127.0.0.1" - remote_logname: "bob" - auth_user: "frank" - timestamp: "2000-10-10T20:55:36Z" - request_line: "GET /apache_pb.gif HTTP/1.0" - request_method: "GET" - request_path: "/apache_pb.gif" - request_protocol: "HTTP/1.0" - status_code: 200 - content_length: 2326 + host: "127.0.0.1" + identity: "bob" + user: "frank" + timestamp: "2000-10-10T20:55:36Z" + message: "GET /apache_pb.gif HTTP/1.0" + method: "GET" + path: "/apache_pb.gif" + protocol: "HTTP/1.0" + status: 200 + size: 2326 } }, ] diff --git a/lib/remap-functions/src/parse_common_log.rs b/lib/remap-functions/src/parse_common_log.rs index 5d44a31f310ed..4d5253ed5d0dc 100644 --- a/lib/remap-functions/src/parse_common_log.rs +++ b/lib/remap-functions/src/parse_common_log.rs @@ -11,19 +11,19 @@ lazy_static! { static ref REGEX_COMMON_LOG: Regex = Regex::new( r#"(?x) # Ignore whitespace and comments in the regex expression. ^\s* # Start with any number of whitespaces. - (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. - (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. - (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. + (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. + (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. + (-|(?P.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace. (-|\[(-|(?P[^\[]*))\])\s+ # Match `-` or `[` followed by `-` or any character except `]`, `]` and at least one whitespace. (-|"(-|(\s* # Match `-` or `"` followed by `-` or and any number of whitespaces... - (?P( # Match a request with... - (?P\w+)\s+ # Match at least one word character and at least one whitespace. - (?P[[\\"][^"]]*?)\s+ # Match any character except `"`, but `\"` (non-greedily) and at least one whitespace. - (?P[[\\"][^"]]*?)\s* # Match any character except `"`, but `\"` (non-greedily) and any number of whitespaces. + (?P( # Match a request with... + (?P\w+)\s+ # Match at least one word character and at least one whitespace. + (?P[[\\"][^"]]*?)\s+ # Match any character except `"`, but `\"` (non-greedily) and at least one whitespace. + (?P[[\\"][^"]]*?)\s* # Match any character except `"`, but `\"` (non-greedily) and any number of whitespaces. |[[\\"][^"]]*?))\s*))" # ...Or match any charater except `"`, but `\"`, and any amount of whitespaces. )\s+ # Match at least one whitespace. - (-|(?P\d+))\s+ # Match `-` or at least one digit and at least one whitespace. - (-|(?P\d+)) # Match `-` or at least one digit. + (-|(?P\d+))\s+ # Match `-` or at least one digit and at least one whitespace. + (-|(?P\d+)) # Match `-` or at least one digit. \s* # Match and any number of whitespaces. "#) .expect("failed compiling regex for common log"); @@ -89,28 +89,16 @@ impl Expression for ParseCommonLogFn { .captures(&message) .ok_or("failed parsing common log line")?; - if let Some(remote_host) = captures.name("remote_host").map(|capture| capture.as_str()) { - log.insert( - "remote_host".into(), - Value::Bytes(remote_host.to_owned().into()), - ); + if let Some(host) = captures.name("host").map(|capture| capture.as_str()) { + log.insert("host".into(), Value::Bytes(host.to_owned().into())); } - if let Some(remote_logname) = captures - .name("remote_logname") - .map(|capture| capture.as_str()) - { - log.insert( - "remote_logname".into(), - Value::Bytes(remote_logname.to_owned().into()), - ); + if let Some(identity) = captures.name("identity").map(|capture| capture.as_str()) { + log.insert("identity".into(), Value::Bytes(identity.to_owned().into())); } - if let Some(auth_user) = captures.name("auth_user").map(|capture| capture.as_str()) { - log.insert( - "auth_user".into(), - Value::Bytes(auth_user.to_owned().into()), - ); + if let Some(user) = captures.name("user").map(|capture| capture.as_str()) { + log.insert("user".into(), Value::Bytes(user.to_owned().into())); } if let Some(timestamp) = captures.name("timestamp").map(|capture| capture.as_str()) { @@ -129,68 +117,33 @@ impl Expression for ParseCommonLogFn { ); } - if let Some(request_line) = captures - .name("request_line") - .map(|capture| capture.as_str()) - { - log.insert( - "request_line".into(), - Value::Bytes(request_line.to_owned().into()), - ); + if let Some(message) = captures.name("message").map(|capture| capture.as_str()) { + log.insert("message".into(), Value::Bytes(message.to_owned().into())); } - if let Some(request_method) = captures - .name("request_method") - .map(|capture| capture.as_str()) - { - log.insert( - "request_method".into(), - Value::Bytes(request_method.to_owned().into()), - ); + if let Some(method) = captures.name("method").map(|capture| capture.as_str()) { + log.insert("method".into(), Value::Bytes(method.to_owned().into())); } - if let Some(request_path) = captures - .name("request_path") - .map(|capture| capture.as_str()) - { - log.insert( - "request_path".into(), - Value::Bytes(request_path.to_owned().into()), - ); + if let Some(path) = captures.name("path").map(|capture| capture.as_str()) { + log.insert("path".into(), Value::Bytes(path.to_owned().into())); } - if let Some(request_protocol) = captures - .name("request_protocol") - .map(|capture| capture.as_str()) - { - log.insert( - "request_protocol".into(), - Value::Bytes(request_protocol.to_owned().into()), - ); + if let Some(protocol) = captures.name("protocol").map(|capture| capture.as_str()) { + log.insert("protocol".into(), Value::Bytes(protocol.to_owned().into())); } - if let Some(status_code) = captures.name("status_code").map(|capture| capture.as_str()) { + if let Some(status) = captures.name("status").map(|capture| capture.as_str()) { log.insert( - "status_code".into(), - Value::Integer( - status_code - .parse() - .map_err(|_| "failed parsing status code")?, - ), + "status".into(), + Value::Integer(status.parse().map_err(|_| "failed parsing status code")?), ); } - if let Some(content_length) = captures - .name("content_length") - .map(|capture| capture.as_str()) - { + if let Some(size) = captures.name("size").map(|capture| capture.as_str()) { log.insert( - "content_length".into(), - Value::Integer( - content_length - .parse() - .map_err(|_| "failed parsing content length")?, - ), + "size".into(), + Value::Integer(size.parse().map_err(|_| "failed parsing content length")?), ); } @@ -216,16 +169,16 @@ mod tests { log_line_valid { args: func_args![value: r#"127.0.0.1 bob frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326"#], want: Ok(btreemap! { - "remote_host" => "127.0.0.1", - "remote_logname" => "bob", - "auth_user" => "frank", + "host" => "127.0.0.1", + "identity" => "bob", + "user" => "frank", "timestamp" => Value::Timestamp(DateTime::parse_from_rfc3339("2000-10-10T20:55:36Z").unwrap().into()), - "request_line" => "GET /apache_pb.gif HTTP/1.0", - "request_method" => "GET", - "request_path" => "/apache_pb.gif", - "request_protocol" => "HTTP/1.0", - "status_code" => 200, - "content_length" => 2326, + "message" => "GET /apache_pb.gif HTTP/1.0", + "method" => "GET", + "path" => "/apache_pb.gif", + "protocol" => "HTTP/1.0", + "status" => 200, + "size" => 2326, }), }