Skip to content

Commit

Permalink
Make field names of common log message more concise
Browse files Browse the repository at this point in the history 
Signed-off-by: Pablo Sichert <[email protected]>
  • Loading branch information
@pablosichert
pablosichert committed Jan 30, 2021
1 parent 6e448af commit fc82d00
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 105 deletions.
40 changes: 20 additions & 20 deletions docs/reference/remap/functions/parse_common_log.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,16 @@ remap: functions: parse_common_log: {
parse_common_log("127.0.0.1 bob frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326")
"""#
return: {
remote_host: "127.0.0.1"
remote_logname: "bob"
auth_user: "frank"
timestamp: "2000-10-10T20:55:36Z"
request_line: "GET /apache_pb.gif HTTP/1.0"
request_method: "GET"
request_path: "/apache_pb.gif"
request_protocol: "HTTP/1.0"
status_code: 200
content_length: 2326
host: "127.0.0.1"
identity: "bob"
user: "frank"
timestamp: "2000-10-10T20:55:36Z"
message: "GET /apache_pb.gif HTTP/1.0"
method: "GET"
path: "/apache_pb.gif"
protocol: "HTTP/1.0"
status: 200
size: 2326
}
},
{
Expand All @@ -61,16 +61,16 @@ remap: functions: parse_common_log: {
)
"""#
return: {
remote_host: "127.0.0.1"
remote_logname: "bob"
auth_user: "frank"
timestamp: "2000-10-10T20:55:36Z"
request_line: "GET /apache_pb.gif HTTP/1.0"
request_method: "GET"
request_path: "/apache_pb.gif"
request_protocol: "HTTP/1.0"
status_code: 200
content_length: 2326
host: "127.0.0.1"
identity: "bob"
user: "frank"
timestamp: "2000-10-10T20:55:36Z"
message: "GET /apache_pb.gif HTTP/1.0"
method: "GET"
path: "/apache_pb.gif"
protocol: "HTTP/1.0"
status: 200
size: 2326
}
},
]
Expand Down
123 changes: 38 additions & 85 deletions lib/remap-functions/src/parse_common_log.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ lazy_static! {
static ref REGEX_COMMON_LOG: Regex = Regex::new(
r#"(?x) # Ignore whitespace and comments in the regex expression.
^\s* # Start with any number of whitespaces.
(-|(?P<remote_host>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|(?P<remote_logname>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|(?P<auth_user>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|(?P<host>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|(?P<identity>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|(?P<user>.*?))\s+ # Match `-` or any character (non-greedily) and at least one whitespace.
(-|\[(-|(?P<timestamp>[^\[]*))\])\s+ # Match `-` or `[` followed by `-` or any character except `]`, `]` and at least one whitespace.
(-|"(-|(\s* # Match `-` or `"` followed by `-` or and any number of whitespaces...
(?P<request_line>( # Match a request with...
(?P<request_method>\w+)\s+ # Match at least one word character and at least one whitespace.
(?P<request_path>[[\\"][^"]]*?)\s+ # Match any character except `"`, but `\"` (non-greedily) and at least one whitespace.
(?P<request_protocol>[[\\"][^"]]*?)\s* # Match any character except `"`, but `\"` (non-greedily) and any number of whitespaces.
(?P<message>( # Match a request with...
(?P<method>\w+)\s+ # Match at least one word character and at least one whitespace.
(?P<path>[[\\"][^"]]*?)\s+ # Match any character except `"`, but `\"` (non-greedily) and at least one whitespace.
(?P<protocol>[[\\"][^"]]*?)\s* # Match any character except `"`, but `\"` (non-greedily) and any number of whitespaces.
|[[\\"][^"]]*?))\s*))" # ...Or match any charater except `"`, but `\"`, and any amount of whitespaces.
)\s+ # Match at least one whitespace.
(-|(?P<status_code>\d+))\s+ # Match `-` or at least one digit and at least one whitespace.
(-|(?P<content_length>\d+)) # Match `-` or at least one digit.
(-|(?P<status>\d+))\s+ # Match `-` or at least one digit and at least one whitespace.
(-|(?P<size>\d+)) # Match `-` or at least one digit.
\s* # Match and any number of whitespaces.
"#)
.expect("failed compiling regex for common log");
Expand Down Expand Up @@ -89,28 +89,16 @@ impl Expression for ParseCommonLogFn {
.captures(&message)
.ok_or("failed parsing common log line")?;

if let Some(remote_host) = captures.name("remote_host").map(|capture| capture.as_str()) {
log.insert(
"remote_host".into(),
Value::Bytes(remote_host.to_owned().into()),
);
if let Some(host) = captures.name("host").map(|capture| capture.as_str()) {
log.insert("host".into(), Value::Bytes(host.to_owned().into()));
}

if let Some(remote_logname) = captures
.name("remote_logname")
.map(|capture| capture.as_str())
{
log.insert(
"remote_logname".into(),
Value::Bytes(remote_logname.to_owned().into()),
);
if let Some(identity) = captures.name("identity").map(|capture| capture.as_str()) {
log.insert("identity".into(), Value::Bytes(identity.to_owned().into()));
}

if let Some(auth_user) = captures.name("auth_user").map(|capture| capture.as_str()) {
log.insert(
"auth_user".into(),
Value::Bytes(auth_user.to_owned().into()),
);
if let Some(user) = captures.name("user").map(|capture| capture.as_str()) {
log.insert("user".into(), Value::Bytes(user.to_owned().into()));
}

if let Some(timestamp) = captures.name("timestamp").map(|capture| capture.as_str()) {
Expand All @@ -129,68 +117,33 @@ impl Expression for ParseCommonLogFn {
);
}

if let Some(request_line) = captures
.name("request_line")
.map(|capture| capture.as_str())
{
log.insert(
"request_line".into(),
Value::Bytes(request_line.to_owned().into()),
);
if let Some(message) = captures.name("message").map(|capture| capture.as_str()) {
log.insert("message".into(), Value::Bytes(message.to_owned().into()));
}

if let Some(request_method) = captures
.name("request_method")
.map(|capture| capture.as_str())
{
log.insert(
"request_method".into(),
Value::Bytes(request_method.to_owned().into()),
);
if let Some(method) = captures.name("method").map(|capture| capture.as_str()) {
log.insert("method".into(), Value::Bytes(method.to_owned().into()));
}

if let Some(request_path) = captures
.name("request_path")
.map(|capture| capture.as_str())
{
log.insert(
"request_path".into(),
Value::Bytes(request_path.to_owned().into()),
);
if let Some(path) = captures.name("path").map(|capture| capture.as_str()) {
log.insert("path".into(), Value::Bytes(path.to_owned().into()));
}

if let Some(request_protocol) = captures
.name("request_protocol")
.map(|capture| capture.as_str())
{
log.insert(
"request_protocol".into(),
Value::Bytes(request_protocol.to_owned().into()),
);
if let Some(protocol) = captures.name("protocol").map(|capture| capture.as_str()) {
log.insert("protocol".into(), Value::Bytes(protocol.to_owned().into()));
}

if let Some(status_code) = captures.name("status_code").map(|capture| capture.as_str()) {
if let Some(status) = captures.name("status").map(|capture| capture.as_str()) {
log.insert(
"status_code".into(),
Value::Integer(
status_code
.parse()
.map_err(|_| "failed parsing status code")?,
),
"status".into(),
Value::Integer(status.parse().map_err(|_| "failed parsing status code")?),
);
}

if let Some(content_length) = captures
.name("content_length")
.map(|capture| capture.as_str())
{
if let Some(size) = captures.name("size").map(|capture| capture.as_str()) {
log.insert(
"content_length".into(),
Value::Integer(
content_length
.parse()
.map_err(|_| "failed parsing content length")?,
),
"size".into(),
Value::Integer(size.parse().map_err(|_| "failed parsing content length")?),
);
}

Expand All @@ -216,16 +169,16 @@ mod tests {
log_line_valid {
args: func_args![value: r#"127.0.0.1 bob frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326"#],
want: Ok(btreemap! {
"remote_host" => "127.0.0.1",
"remote_logname" => "bob",
"auth_user" => "frank",
"host" => "127.0.0.1",
"identity" => "bob",
"user" => "frank",
"timestamp" => Value::Timestamp(DateTime::parse_from_rfc3339("2000-10-10T20:55:36Z").unwrap().into()),
"request_line" => "GET /apache_pb.gif HTTP/1.0",
"request_method" => "GET",
"request_path" => "/apache_pb.gif",
"request_protocol" => "HTTP/1.0",
"status_code" => 200,
"content_length" => 2326,
"message" => "GET /apache_pb.gif HTTP/1.0",
"method" => "GET",
"path" => "/apache_pb.gif",
"protocol" => "HTTP/1.0",
"status" => 200,
"size" => 2326,
}),
}

Expand Down

0 comments on commit fc82d00

Please sign in to comment.