Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick #20370 to 7.x: Check expand_event_list_from_field when json in map[string]interface{} format #20374

Merged
merged 1 commit into from
Jul 31, 2020
+24 −1
Merged

Cherry-pick #20370 to 7.x: Check expand_event_list_from_field when json in map[string]interface{} format #20374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962]
- Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984]
- Fix s3 input parsing json file without expand_event_list_from_field. {issue}19902[19902] {pull}19962[19962] {pull}20370[20370]
- Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
- Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
- Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
Expand Down
24 changes: 23 additions & 1 deletion x-pack/filebeat/input/s3/input.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,10 +526,11 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3
return nil
}

offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx)
offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx)
if err != nil {
return err
}
offset = offsetNew
}
}

Expand All @@ -554,6 +555,27 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash
return offset, nil
}
case map[string]interface{}:
if p.config.ExpandEventListFromField != "" {
textValues, ok := f[p.config.ExpandEventListFromField]
if !ok {
err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField)
p.logger.Error(err)
return offset, err
}

valuesConverted := textValues.([]interface{})
for _, textValue := range valuesConverted {
offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx)
if err != nil {
err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name)
p.logger.Error(err)
return offset, err
}
offset = offsetNew
}
return offset, nil
}

offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx)
if err != nil {
err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name)
Expand Down