From 5dfeb7b9039686be6a9860c6bb08b50cdd7576d5 Mon Sep 17 00:00:00 2001 From: P1llus Date: Tue, 6 Oct 2020 18:25:51 +0200 Subject: [PATCH 1/4] adding fips support for s3 input --- x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc | 5 +++++ x-pack/filebeat/input/s3/config.go | 2 ++ x-pack/filebeat/input/s3/input.go | 7 ++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 8891e38fcc4..c4dda58b912 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -42,6 +42,11 @@ The `s3` input supports the following configuration options plus the URL of the AWS SQS queue that messages will be received from. Required. +[float] +==== `fips_enabled` + +When enabled this option changes the servicename from s3 to s3-fips for connecting to the correct s3 endpoint + [float] ==== `visibility_timeout` diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index 5f37a436d12..cc3c5318289 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -15,6 +15,7 @@ import ( type config struct { QueueURL string `config:"queue_url" validate:"nonzero,required"` VisibilityTimeout time.Duration `config:"visibility_timeout"` + FipsEnabled bool `config:"fips_enabled"` AwsConfig awscommon.ConfigAWS `config:",inline"` ExpandEventListFromField string `config:"expand_event_list_from_field"` APITimeout time.Duration `config:"api_timeout"` @@ -32,6 +33,7 @@ func defaultConfig() config { return config{ VisibilityTimeout: 300 * time.Second, APITimeout: 120 * time.Second, + FipsEnabled: false, } } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index a6b56d03970..fb1512afe9f 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -100,6 +100,11 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C log.Infof("visibility timeout is set to %v seconds", visibilityTimeout) log.Infof("aws api timeout is set to %v", in.config.APITimeout) + s3Servicename := "s3" + if in.config.FipsEnabled { + s3Servicename = "s3-fips" + } + return &s3Collector{ cancellation: ctxtool.FromCanceller(ctx.Cancelation), logger: log, @@ -107,7 +112,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C publisher: client, visibilityTimeout: visibilityTimeout, sqs: sqs.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)), - s3: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)), + s3: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, s3Servicename, regionName, awsConfig)), }, nil } From 86f5e52e7cffece98987424a83c044bd2f85ce9d Mon Sep 17 00:00:00 2001 From: P1llus Date: Tue, 6 Oct 2020 18:51:04 +0200 Subject: [PATCH 2/4] adding changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 99daa875a00..6d22c4c1bf9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -612,6 +612,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - New Cisco Umbrella dataset {pull}21504[21504] - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017] - Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446] +- Adding support for FIPS in s3 input {pull}21446[21446] *Heartbeat* From ba9cf5c6c35e2ec97baa2664bfb17c564cd3ded8 Mon Sep 17 00:00:00 2001 From: P1llus Date: Tue, 6 Oct 2020 18:53:17 +0200 Subject: [PATCH 3/4] updating docs based on PR comments --- x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index c4dda58b912..5cbe4685cb8 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -45,7 +45,7 @@ URL of the AWS SQS queue that messages will be received from. Required. [float] ==== `fips_enabled` -When enabled this option changes the servicename from s3 to s3-fips for connecting to the correct s3 endpoint +Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. For example: `s3-fips.us-gov-east-1.amazonaws.com`. [float] ==== `visibility_timeout` From 54b048880e95f845fb9a6f36786d4ee90cb3f22b Mon Sep 17 00:00:00 2001 From: P1llus Date: Tue, 6 Oct 2020 19:02:46 +0200 Subject: [PATCH 4/4] adding debug log for s3 input --- x-pack/filebeat/input/s3/input.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index fb1512afe9f..d76e5b8b728 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -105,6 +105,8 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C s3Servicename = "s3-fips" } + log.Debug("s3 service name = ", s3Servicename) + return &s3Collector{ cancellation: ctxtool.FromCanceller(ctx.Cancelation), logger: log,