From 326f6612878d6f3a6e8fb753cdae8d6369ba57c4 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 10 Jan 2020 10:38:55 +0100 Subject: [PATCH] NetFlow: Allow for zero scope fields in options template NetFlow v9 spec allows for options templates that contain no scope fields. The netflow input was treating this case as an error and discarding the template, but that is only applicable to IPFIX. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go | 1 + .../filebeat/input/netflow/decoder/template/template.go | 5 ++++- .../input/netflow/decoder/template/template_test.go | 8 ++++++++ x-pack/filebeat/input/netflow/decoder/v9/decoder.go | 3 ++- 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 688b530b89e5..0a6c7f4b3a3d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -230,6 +230,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix SSL config in input.yml for Filebeat httpjson input in the MISP module. {pull}14767[14767] - Check content-type when creating new reader in s3 input. {pull}15252[15252] {issue}15225[15225] - Fix session reset detection and a crash in Netflow input. {pull}14904[14904] +- netflow v9: Allow for options templates without scope fields. {pull}15449[15449] *Heartbeat* diff --git a/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go b/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go index 664837755211..9c0252cb9c4e 100644 --- a/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go +++ b/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go @@ -108,6 +108,7 @@ func (d DecoderIPFIX) ReadOptionsTemplateFlowSet(buf *bytes.Buffer) (templates [ } template.ID = tID template.ScopeFields = scopeCount + template.IsOptions = true templates = append(templates, &template) } return templates, nil diff --git a/x-pack/filebeat/input/netflow/decoder/template/template.go b/x-pack/filebeat/input/netflow/decoder/template/template.go index dc42bcc26f07..e04dc93cfe36 100644 --- a/x-pack/filebeat/input/netflow/decoder/template/template.go +++ b/x-pack/filebeat/input/netflow/decoder/template/template.go @@ -29,6 +29,9 @@ type Template struct { Length int VariableLength bool ScopeFields int + // IsOptions signals that this is an options template. Previously + // ScopeFields>0 was used for this, but that's unreliable under v9. + IsOptions bool } type FieldTemplate struct { @@ -84,7 +87,7 @@ func (t *Template) Apply(data *bytes.Buffer, n int) ([]record.Record, error) { } } makeFn := t.makeFlow - if t.ScopeFields > 0 { + if t.IsOptions { makeFn = t.makeOptions } events := make([]record.Record, 0, alloc) diff --git a/x-pack/filebeat/input/netflow/decoder/template/template_test.go b/x-pack/filebeat/input/netflow/decoder/template/template_test.go index 6ab22766170e..882756dc9a2a 100644 --- a/x-pack/filebeat/input/netflow/decoder/template/template_test.go +++ b/x-pack/filebeat/input/netflow/decoder/template/template_test.go @@ -313,6 +313,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 7, ScopeFields: 1, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: 2, Info: &fields.Field{Name: "destinationTransportPort", Decoder: fields.Unsigned16}}, @@ -343,6 +344,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 7, ScopeFields: 2, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: 2, Info: &fields.Field{Name: "destinationTransportPort", Decoder: fields.Unsigned16}}, @@ -386,6 +388,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 7, ScopeFields: 3, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: 2, Info: &fields.Field{Name: "destinationTransportPort", Decoder: fields.Unsigned16}}, @@ -415,6 +418,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 7, ScopeFields: 1, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: 2, Info: &fields.Field{Name: "destinationTransportPort", Decoder: fields.Unsigned16}}, @@ -446,6 +450,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 7, ScopeFields: 2, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: 2, Info: &fields.Field{Name: "destinationTransportPort", Decoder: fields.Unsigned16}}, @@ -489,6 +494,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 6, ScopeFields: 1, + IsOptions: true, VariableLength: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, @@ -522,6 +528,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { record: Template{ Length: 6, ScopeFields: 1, + IsOptions: true, VariableLength: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, @@ -571,6 +578,7 @@ func TestOptionsTemplate_Apply(t *testing.T) { Length: 6, VariableLength: true, ScopeFields: 2, + IsOptions: true, Fields: []FieldTemplate{ {Length: 4, Info: &fields.Field{Name: "sourceIPv4Address", Decoder: fields.Ipv4Address}}, {Length: VariableLength, Info: &fields.Field{Name: "vpnIdentifier", Decoder: fields.OctetArray}}, diff --git a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go index f59726846525..6b4fc9be8be7 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go @@ -184,7 +184,7 @@ func (d DecoderV9) ReadOptionsTemplateFlowSet(buf *bytes.Buffer) (templates []*t if buf.Len() < int(length) { return nil, io.EOF } - if scopeLen == 0 || scopeLen&3 != 0 || optsLen&3 != 0 { + if (scopeLen+optsLen) == 0 || scopeLen&3 != 0 || optsLen&3 != 0 { return nil, fmt.Errorf("bad length for options template. scope=%d options=%d", scopeLen, optsLen) } template, err := ReadFields(d, buf, (scopeLen+optsLen)/4) @@ -193,6 +193,7 @@ func (d DecoderV9) ReadOptionsTemplateFlowSet(buf *bytes.Buffer) (templates []*t } template.ID = tID template.ScopeFields = scopeLen / 4 + template.IsOptions = true templates = append(templates, &template) } return templates, nil