Skip to content

Commit

Permalink
Add line number counter for multiline
Browse files Browse the repository at this point in the history
This allows to indicate if an event was multiline or not. The number of lines will be put under the multiline namespace and looks as following:

```
{
  ...
  "message": "[2015] hello world\n  First Line\n  Second Line",
  "multiline": {
    "lines": 3
  },
  ...
}
```

See elastic#957
  • Loading branch information
ruflin committed Sep 25, 2017
1 parent 5ecba0f commit 1865dcf
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 0 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,11 @@ https://github.com/elastic/beats/compare/v5.2.0...v5.2.1[View commits]
*Packetbeat*
- Fix error in the NFS sample dashboard. {pull}3548[3548]
- Add enabled config option to prospectors. {pull}3157[3157]
- Add target option for decoded_json_field. {pull}3169[3169]
- Add the `pipeline` config option at the prospector level, for configuring the Ingest Node pipeline ID. {pull}3433[3433]
- Update regular expressions used for matching file names or lines (multiline, include/exclude functionality) to new matchers improving performance of simple string matches. {pull}3469[3469]
- Add multiline line number to filebeat event {pull}2279[2279]
*Winlogbeat*
Expand Down
5 changes: 5 additions & 0 deletions filebeat/_meta/fields.common.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,8 @@
- name: fileset.name
description: >
The Filebeat fileset that generated this event.
- name: multiline.lines
type: long
description: >
This is set in case of a multiline event and contains the number of lines that the multiline event consists of.
8 changes: 8 additions & 0 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -959,6 +959,14 @@ The Filebeat module that generated this event.
The Filebeat fileset that generated this event.
[float]
=== `multiline.lines`
type: long
This is set in case of a multiline event and contains the number of lines that the multiline event consists of.
[[exported-fields-mysql]]
== MySQL fields
Expand Down
7 changes: 7 additions & 0 deletions filebeat/harvester/reader/multiline.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"time"

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/match"
"github.com/elastic/beats/libbeat/logp"
)
Expand Down Expand Up @@ -248,6 +249,12 @@ func (mlr *Multiline) clear() {

// finalize writes the existing content into the returned message and resets all reader variables.
func (mlr *Multiline) finalize() Message {

mlr.message.AddFields(common.MapStr{
"multiline": common.MapStr{
"lines": mlr.numLines,
},
})
// Copy message from existing content
msg := mlr.message
mlr.clear()
Expand Down
33 changes: 33 additions & 0 deletions filebeat/tests/system/test_multiline.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,3 +352,36 @@ def test_invalid_config(self):
self.wait_until(lambda: self.log_contains("missing required field accessing") == 1)

proc.check_kill_and_wait(exit_code=1)

def test_multiline_lines(self):
"""
Test if multiline line counter is added
"""
self.render_config_template(
path=os.path.abspath(self.working_dir) + "/log/*",
multiline=True,
pattern="^\[",
negate="true",
match="after",
close_timeout="1s",
)

os.mkdir(self.working_dir + "/log/")

testfile = self.working_dir + "/log/test.log"

with open(testfile, 'w', 0) as file:
file.write("[2015] hello world\n")
file.write(" First Line\n")
file.write(" Second Line\n")

filebeat = self.start_beat()

self.wait_until(
lambda: self.output_has(lines=1),
max_timeout=10)

filebeat.check_kill_and_wait()

output = self.read_output_json()
output[0]["multiline"]["lines"] = 3

0 comments on commit 1865dcf

Please sign in to comment.