From e5b1ba7aae84c19d937b6b68051283d403b794fc Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 14 Oct 2021 17:00:34 -0600 Subject: [PATCH 1/3] Check when response content type is nil --- x-pack/filebeat/input/awss3/s3_objects.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index 66590092baa6..3b550a089979 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -174,10 +174,13 @@ func (p *s3ObjectProcessor) download() (contentType string, metadata map[string] } if resp == nil { - return "", nil, nil, errors.New("empty reponse from s3 get object") + return "", nil, nil, errors.New("empty response from s3 get object") } meta := s3Metadata(resp, p.readerConfig.IncludeS3Metadata...) + if resp.ContentType == nil { + return "", meta, resp.Body, nil + } return *resp.ContentType, meta, resp.Body, nil } From 308bc12f3fe0d02955347748868bac852050af33 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 14 Oct 2021 17:06:13 -0600 Subject: [PATCH 2/3] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d874adfe019b..30e20401f70a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -320,6 +320,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325] - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352] - Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277] +- Fix aws-s3 input by checking if GetObject API call response content type exists. {pull}28457[28457] *Heartbeat* From 6cf48ffdb180cec319a556d9c6ca3d66f8b86fa7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 15 Oct 2021 10:34:45 -0600 Subject: [PATCH 3/3] add unit test --- .../filebeat/input/awss3/s3_objects_test.go | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/awss3/s3_objects_test.go b/x-pack/filebeat/input/awss3/s3_objects_test.go index 4b664a666f23..04b6848f1eeb 100644 --- a/x-pack/filebeat/input/awss3/s3_objects_test.go +++ b/x-pack/filebeat/input/awss3/s3_objects_test.go @@ -40,9 +40,13 @@ func newS3GetObjectResponse(filename string, data []byte, contentType string) *s GetObjectOutput: &s3.GetObjectOutput{ Body: ioutil.NopCloser(r), ContentLength: &contentLen, - ContentType: &contentType, }, } + + if contentType != "" { + resp.ContentType = &contentType + } + switch strings.ToLower(filepath.Ext(filename)) { case ".gz": gzipEncoding := "gzip" @@ -169,6 +173,33 @@ func TestS3ObjectProcessor(t *testing.T) { err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), ack, s3Event).ProcessS3Object() require.Error(t, err) }) + + t.Run("no content type in GetObject response", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + ctrl, ctx := gomock.WithContext(ctx, t) + defer ctrl.Finish() + mockS3API := NewMockS3API(ctrl) + mockPublisher := NewMockBeatClient(ctrl) + s3Event, s3Resp := newS3Object(t, "testdata/log.txt", "") + + var events []beat.Event + gomock.InOrder( + mockS3API.EXPECT(). + GetObject(gomock.Any(), gomock.Eq(s3Event.S3.Bucket.Name), gomock.Eq(s3Event.S3.Object.Key)). + Return(s3Resp, nil), + mockPublisher.EXPECT(). + Publish(gomock.Any()). + Do(func(event beat.Event) { events = append(events, event) }). + Times(2), + ) + + s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, mockPublisher, nil) + ack := newEventACKTracker(ctx) + err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), ack, s3Event).ProcessS3Object() + require.NoError(t, err) + }) } func testProcessS3Object(t testing.TB, file, contentType string, numEvents int, selectors ...fileSelectorConfig) []beat.Event {