From 7e7bc2e6b1f178c4677c50cadb25a7ebeedf8ef5 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 26 Feb 2020 16:24:03 -0500 Subject: [PATCH] Reduce memory allocations in decode_cef processor (#16587) * Add decode_cef processor benchmarks * Take advantage of string slicing to avoid allocations Change the input of the cef parser to accept a string instead of []byte to avoid unnecessary copies. Data is still copied from the input message when it contains escape sequences. Another minor improvement is to allocate the map for the CEF extensions up front based on the estimated number of extension fields. Results from `go test -run none -bench . -benchtime 5s -benchmem .` before and after: $ benchcmp before.txt after.txt benchmark old ns/op new ns/op delta BenchmarkProcessorRun/short_msg-12 4833 4684 -3.08% BenchmarkProcessorRun/long_msg-12 55724 52493 -5.80% benchmark old allocs new allocs delta BenchmarkProcessorRun/short_msg-12 55 41 -25.45% BenchmarkProcessorRun/long_msg-12 349 219 -37.25% benchmark old bytes new bytes delta BenchmarkProcessorRun/short_msg-12 3728 3424 -8.15% BenchmarkProcessorRun/long_msg-12 26929 21173 -21.37% --- CHANGELOG.next.asciidoc | 1 + .../filebeat/processors/decode_cef/cef/cef.go | 55 ++++++++++++------- .../filebeat/processors/decode_cef/cef/cef.rl | 26 ++++----- .../processors/decode_cef/cef/cef_test.go | 38 ++++++------- .../decode_cef/cef/cmd/cef2json/cef2json.go | 2 +- .../processors/decode_cef/cef/fuzz/fuzz.go | 2 +- .../processors/decode_cef/cef/parser.go | 42 +++++++------- .../processors/decode_cef/decode_cef.go | 2 +- .../processors/decode_cef/decode_cef_test.go | 35 ++++++++++++ 9 files changed, 126 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c5a2e2046d49..9327d3322e05 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500] - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529] +- Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587] *Heartbeat* diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.go b/x-pack/filebeat/processors/decode_cef/cef/cef.go index c3cee5ff02d4..2ca1c95758a1 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.go @@ -5,7 +5,7 @@ package cef import ( - "bytes" + "strings" "github.com/pkg/errors" "go.uber.org/multierr" @@ -59,7 +59,7 @@ type Event struct { Extensions map[string]*Field `json:"extensions,omitempty"` } -func (e *Event) init() { +func (e *Event) init(data string) { e.Version = -1 e.DeviceVendor = "" e.DeviceProduct = "" @@ -68,13 +68,25 @@ func (e *Event) init() { e.Name = "" e.Severity = "" e.Extensions = nil + + // Estimate length of the extensions. But limit the allocation because + // it's based on user input. This doesn't account for escaped equals. + if n := strings.Count(data, "="); n > 0 { + const maxLen = 50 + if n <= maxLen { + e.Extensions = make(map[string]*Field, n) + } else { + e.Extensions = make(map[string]*Field, maxLen) + } + } } -func (e *Event) pushExtension(key []byte, value []byte) { +func (e *Event) pushExtension(key, value string) { if e.Extensions == nil { e.Extensions = map[string]*Field{} } - e.Extensions[string(key)] = &Field{String: string(value)} + field := &Field{String: value} + e.Extensions[key] = field } // Unpack unpacks a common event format (CEF) message. The data is expected to @@ -99,7 +111,7 @@ func (e *Event) pushExtension(key []byte, value []byte) { // and may contain alphanumeric, underscore (_), period (.), comma (,), and // brackets ([) (]). This is less strict than the CEF specification, but aligns // the key names used in practice. -func (e *Event) Unpack(data []byte, opts ...Option) error { +func (e *Event) Unpack(data string, opts ...Option) error { var settings Settings for _, opt := range opts { opt.Apply(&settings) @@ -137,29 +149,32 @@ func (e *Event) Unpack(data []byte, opts ...Option) error { return multierr.Combine(errs...) } -var ( - backslash = []byte(`\`) - escapedBackslash = []byte(`\\`) +const ( + backslash = `\` + escapedBackslash = `\\` - pipe = []byte(`|`) - escapedPipe = []byte(`\|`) + pipe = `|` + escapedPipe = `\|` - equalsSign = []byte(`=`) - escapedEqualsSign = []byte(`\=`) + equalsSign = `=` + escapedEqualsSign = `\=` +) + +var ( + headerEscapes = strings.NewReplacer(escapedBackslash, backslash, escapedPipe, pipe) + extensionEscapes = strings.NewReplacer(escapedBackslash, backslash, escapedEqualsSign, equalsSign) ) -func replaceHeaderEscapes(b []byte) []byte { - if bytes.IndexByte(b, '\\') != -1 { - b = bytes.ReplaceAll(b, escapedBackslash, backslash) - b = bytes.ReplaceAll(b, escapedPipe, pipe) +func replaceHeaderEscapes(b string) string { + if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedPipe) != -1 { + return headerEscapes.Replace(b) } return b } -func replaceExtensionEscapes(b []byte) []byte { - if bytes.IndexByte(b, '\\') != -1 { - b = bytes.ReplaceAll(b, escapedBackslash, backslash) - b = bytes.ReplaceAll(b, escapedEqualsSign, equalsSign) +func replaceExtensionEscapes(b string) string { + if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedEqualsSign) != -1 { + return extensionEscapes.Replace(b) } return b } diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.rl b/x-pack/filebeat/processors/decode_cef/cef/cef.rl index 55248ab1534b..3ac5af35a408 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.rl +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.rl @@ -16,12 +16,12 @@ import ( }%% // unpack unpacks a CEF message. -func (e *Event) unpack(data []byte) error { +func (e *Event) unpack(data string) error { cs, p, pe, eof := 0, 0, len(data), len(data) mark := 0 // Extension key. - var extKey []byte + var extKey string // Extension value start and end indices. extValueStart, extValueEnd := 0, 0 @@ -30,7 +30,7 @@ func (e *Event) unpack(data []byte) error { // recover from (though the parsing might not be "correct"). var recoveredErrs []error - e.init() + e.init(data) %%{ # Actions to execute while executing state machine. @@ -38,31 +38,31 @@ func (e *Event) unpack(data []byte) error { mark = p } action version { - e.Version, _ = strconv.Atoi(string(data[mark:p])) + e.Version, _ = strconv.Atoi(data[mark:p]) } action device_vendor { - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) } action device_product { - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) } action device_version { - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) } action device_event_class_id { - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) } action name { - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) } action severity { - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] } action extension_key { // A new extension key marks the end of the last extension value. if len(extKey) > 0 && extValueStart <= mark - 1 { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } extKey = data[mark:p] } @@ -77,7 +77,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } } action extension_err { @@ -85,7 +85,7 @@ func (e *Event) unpack(data []byte) error { fhold; fgoto gobble_extension; } action recover_next_extension { - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 // Resume processing at p, the start of the next extension key. p = mark; fgoto extensions; diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go index a24e38d3aac3..361b105cd52e 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go @@ -79,7 +79,7 @@ func TestGenerateFuzzCorpus(t *testing.T) { func TestEventUnpack(t *testing.T) { t.Run("standardMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(standardMessage)) + err := e.Unpack(standardMessage) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -98,7 +98,7 @@ func TestEventUnpack(t *testing.T) { t.Run("headerOnly", func(t *testing.T) { var e Event - err := e.Unpack([]byte(headerOnly)) + err := e.Unpack(headerOnly) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -112,7 +112,7 @@ func TestEventUnpack(t *testing.T) { t.Run("escapedPipeInHeader", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapedPipeInHeader)) + err := e.Unpack(escapedPipeInHeader) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -130,7 +130,7 @@ func TestEventUnpack(t *testing.T) { t.Run("equalsSignInHeader", func(t *testing.T) { var e Event - err := e.Unpack([]byte(equalsSignInHeader)) + err := e.Unpack(equalsSignInHeader) assert.NoError(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -148,7 +148,7 @@ func TestEventUnpack(t *testing.T) { t.Run("emptyExtensionValue", func(t *testing.T) { var e Event - err := e.Unpack([]byte(emptyExtensionValue)) + err := e.Unpack(emptyExtensionValue) assert.Error(t, err) assert.Equal(t, 26, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -165,7 +165,7 @@ func TestEventUnpack(t *testing.T) { t.Run("emptyDeviceFields", func(t *testing.T) { var e Event - err := e.Unpack([]byte(emptyDeviceFields)) + err := e.Unpack(emptyDeviceFields) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "", e.DeviceVendor) @@ -183,7 +183,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorEscapedPipeInExtension", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapedPipeInExtension)) + err := e.Unpack(escapedPipeInExtension) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) assert.Equal(t, "threatmanager", e.DeviceProduct) @@ -191,7 +191,7 @@ func TestEventUnpack(t *testing.T) { assert.Equal(t, "100", e.DeviceEventClassID) assert.Equal(t, "trojan successfully stopped", e.Name) assert.Equal(t, "10", e.Severity) - assert.Nil(t, e.Extensions) + assert.Empty(t, e.Extensions) // Pipes in extensions should not be escaped. assert.Error(t, err) @@ -199,7 +199,7 @@ func TestEventUnpack(t *testing.T) { t.Run("leadingWhitespace", func(t *testing.T) { var e Event - err := e.Unpack([]byte(leadingWhitespace)) + err := e.Unpack(leadingWhitespace) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -217,7 +217,7 @@ func TestEventUnpack(t *testing.T) { t.Run("pipeInMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(pipeInMessage)) + err := e.Unpack(pipeInMessage) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -233,7 +233,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorEqualsInMessage", func(t *testing.T) { var e Event - err := e.Unpack([]byte(equalsInMessage)) + err := e.Unpack(equalsInMessage) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) assert.Equal(t, "threatmanager", e.DeviceProduct) @@ -241,7 +241,7 @@ func TestEventUnpack(t *testing.T) { assert.Equal(t, "100", e.DeviceEventClassID) assert.Equal(t, "trojan successfully stopped", e.Name) assert.Equal(t, "10", e.Severity) - assert.Nil(t, e.Extensions) + assert.Empty(t, e.Extensions) // moo contains unescaped equals signs. assert.Error(t, err) @@ -249,7 +249,7 @@ func TestEventUnpack(t *testing.T) { t.Run("escapesInExtension", func(t *testing.T) { var e Event - err := e.Unpack([]byte(escapesInExtension)) + err := e.Unpack(escapesInExtension) assert.NoError(t, err) assert.Equal(t, 0, e.Version) assert.Equal(t, "security", e.DeviceVendor) @@ -266,7 +266,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorMalformedExtensionEscape", func(t *testing.T) { var e Event - err := e.Unpack([]byte(malformedExtensionEscape)) + err := e.Unpack(malformedExtensionEscape) assert.Equal(t, 0, e.Version) assert.Equal(t, "FooBar", e.DeviceVendor) assert.Equal(t, "Web Gateway", e.DeviceProduct) @@ -296,7 +296,7 @@ func TestEventUnpack(t *testing.T) { t.Run("errorMultipleMalformedExtensionValues", func(t *testing.T) { var e Event - err := e.Unpack([]byte(multipleMalformedExtensionValues)) + err := e.Unpack(multipleMalformedExtensionValues) assert.Equal(t, 0, e.Version) assert.Equal(t, "vendor", e.DeviceVendor) assert.Equal(t, "product", e.DeviceProduct) @@ -319,14 +319,14 @@ func TestEventUnpack(t *testing.T) { t.Run("empty", func(t *testing.T) { var e Event - err := e.Unpack([]byte("CEF:0|||||||a=")) + err := e.Unpack("CEF:0|||||||a=") assert.NoError(t, err) }) } func TestEventUnpackWithFullExtensionNames(t *testing.T) { var e Event - err := e.Unpack([]byte(standardMessage), WithFullExtensionNames()) + err := e.Unpack(standardMessage, WithFullExtensionNames()) assert.NoError(t, err) assert.Equal(t, map[string]*Field{ "sourceAddress": IPField("10.0.0.192"), @@ -337,9 +337,9 @@ func TestEventUnpackWithFullExtensionNames(t *testing.T) { } func BenchmarkEventUnpack(b *testing.B) { - var messages [][]byte + var messages []string for _, m := range testMessages { - messages = append(messages, []byte(m)) + messages = append(messages, m) } b.ResetTimer() diff --git a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go index e11b9309eb2b..d95beff8d045 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go @@ -49,7 +49,7 @@ func main() { line = line[begin:] var e cef.Event - if err := e.Unpack(line, opts...); err != nil { + if err := e.Unpack(string(line), opts...); err != nil { log.Println("ERROR:", err, "in:", string(line)) } diff --git a/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go index 427daae679d8..9114c12f84f8 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go +++ b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go @@ -11,7 +11,7 @@ import ( // Fuzz is the entry point that go-fuzz uses to fuzz the parser. func Fuzz(data []byte) int { var e cef2.Event - if err := e.Unpack(data); err != nil { + if err := e.Unpack(string(data)); err != nil { return 1 } return 0 diff --git a/x-pack/filebeat/processors/decode_cef/cef/parser.go b/x-pack/filebeat/processors/decode_cef/cef/parser.go index f7ee7802dad7..2ddcb9424017 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/parser.go +++ b/x-pack/filebeat/processors/decode_cef/cef/parser.go @@ -34,12 +34,12 @@ const cef_en_main_cef_extensions int = 24 //line cef.rl:16 // unpack unpacks a CEF message. -func (e *Event) unpack(data []byte) error { +func (e *Event) unpack(data string) error { cs, p, pe, eof := 0, 0, len(data), len(data) mark := 0 // Extension key. - var extKey []byte + var extKey string // Extension value start and end indices. extValueStart, extValueEnd := 0, 0 @@ -48,7 +48,7 @@ func (e *Event) unpack(data []byte) error { // recover from (though the parsing might not be "correct"). var recoveredErrs []error - e.init() + e.init(data) //line parser.go:55 { @@ -760,43 +760,43 @@ func (e *Event) unpack(data []byte) error { f1: //line cef.rl:40 - e.Version, _ = strconv.Atoi(string(data[mark:p])) + e.Version, _ = strconv.Atoi(data[mark:p]) goto _again f3: //line cef.rl:43 - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) goto _again f5: //line cef.rl:46 - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) goto _again f7: //line cef.rl:49 - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) goto _again f9: //line cef.rl:52 - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) goto _again f11: //line cef.rl:55 - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) goto _again f13: //line cef.rl:58 - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] goto _again f14: @@ -805,7 +805,7 @@ func (e *Event) unpack(data []byte) error { // A new extension key marks the end of the last extension value. if len(extKey) > 0 && extValueStart <= mark-1 { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } extKey = data[mark:p] @@ -829,16 +829,14 @@ func (e *Event) unpack(data []byte) error { recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1)) (p)-- cs = 28 - goto _again f17: //line cef.rl:87 - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 // Resume processing at p, the start of the next extension key. p = mark cs = 24 - goto _again f2: //line cef.rl:37 @@ -847,7 +845,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:43 - e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVendor = replaceHeaderEscapes(data[mark:p]) goto _again f4: @@ -857,7 +855,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:46 - e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceProduct = replaceHeaderEscapes(data[mark:p]) goto _again f6: @@ -867,7 +865,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:49 - e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceVersion = replaceHeaderEscapes(data[mark:p]) goto _again f8: @@ -877,7 +875,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:52 - e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p]) goto _again f10: @@ -887,7 +885,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:55 - e.Name = string(replaceHeaderEscapes(data[mark:p])) + e.Name = replaceHeaderEscapes(data[mark:p]) goto _again f12: @@ -897,7 +895,7 @@ func (e *Event) unpack(data []byte) error { //line cef.rl:58 - e.Severity = string(data[mark:p]) + e.Severity = data[mark:p] goto _again f23: @@ -950,7 +948,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } case 16: @@ -972,7 +970,7 @@ func (e *Event) unpack(data []byte) error { // Reaching the EOF marks the end of the final extension value. if len(extKey) > 0 && extValueStart <= extValueEnd { e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) - extKey, extValueStart, extValueEnd = nil, 0, 0 + extKey, extValueStart, extValueEnd = "", 0, 0 } //line parser.go:847 diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go index 45853f5378a6..ae6672ac024a 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -89,7 +89,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { // If the version < 0 after parsing then none of the data is valid so return here. var ce cef.Event - if err = ce.Unpack([]byte(cefData), cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { + if err = ce.Unpack(cefData, cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { if p.IgnoreFailure { return event, nil } diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef_test.go b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go index a23663861bb3..dd1308fd6786 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go @@ -300,3 +300,38 @@ func assertEqual(t testing.TB, expected, actual interface{}) bool { t.Errorf("Expected and actual are different:\n%s", diff) return false } + +func BenchmarkProcessorRun(b *testing.B) { + dec, err := newDecodeCEF(defaultConfig()) + if err != nil { + b.Fatal(err) + } + + const msg = `CEF:1|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5` + b.Run("short_msg", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := dec.Run(&beat.Event{ + Fields: map[string]interface{}{ + "message": msg, + }, + }) + if err != nil { + b.Fatal(err) + } + } + }) + + const longMsg = `CEF:0|CISCO|ASA||305012|Teardown dynamic UDP translation|Low| eventId=56265798504 mrt=1484092683471 proto=UDP categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Firewall catdt=Firewall categoryOutcome=/Success categoryObject=/Host/Application/Service modelConfidence=0 severity=4 relevance=10 assetCriticality=0 priority=4 art=1484096108163 deviceSeverity=6 rt=1484096094000 src=1.2.3.4 sourceZoneID=GqtK3G9YBABCadQ465CqVeW\=\= sourceZoneURI=/All Zones/GTR/GTR/GTR/GTR sourceTranslatedAddress=4.3.2.1 sourceTranslatedZoneID=P84KXXTYDFYYFwwHq40BQcd\=\= sourceTranslatedZoneURI=/All Zones/GTR/GTR Internet Primary spt=5260 sourceTranslatedPort=5260 cs5=dynamic cs6=0:00:00 c6a4=ffff:0:0:0:222:5555:ffff:5555 locality=1 cs1Label=ACL cs2Label=Unit cs3Label=TCP Flags cs4Label=Order cs5Label=Connection Type cs6Label=Duration cn1Label=ICMP Type cn2Label=ICMP Code cn3Label=DurationInSeconds c6a4Label=Agent IPv6 Address ahost=host.gtr.gtr agt=100.222.333.55 av=7.1.7.7602.0 atz=LA/la aid=4p9IZi1kBABCq5RFPFdJWYUw\=\= at=agent_ac dvchost=super dvc=111.111.111.99 deviceZoneID=K-fU33AAOGVdfFpYAT3UdQ\=\= deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceAssetId=5Wa8hHVSDFBCc-t56wI7mTw\=\= dtz=LA/LA deviceInboundInterface=eth0 deviceOutboundInterface=eth1 eventAnnotationStageUpdateTime=1484097686473 eventAnnotationModificationTime=1484097686475 eventAnnotationAuditTrail=1,1484012146095,root,Queued,,,,\\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1484096094000 eventAnnotationManagerReceiptTime=1484097686471 originalAgentHostName=host originalAgentAddress=10.2.88.3 originalAgentZoneURI=/All Zones/GR/GR/GR originalAgentVersion=7.3.0.7885.0 originalAgentId=6q0sfHVcBABCcSDFvMpvc1w\=\= originalAgentType=syslog_file _cefVer=0.1 ad.arcSightEventPath=7q0sfHVcBABCcMZVvMSDFc1w\=\=` + b.Run("long_msg", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := dec.Run(&beat.Event{ + Fields: map[string]interface{}{ + "message": longMsg, + }, + }) + if err != nil { + b.Fatal(err) + } + } + }) +}