Cherry-pick elastic#15656 to 7.5: [Filebeat] Add support for specifyi…

…ng AWS cred file (elastic#15909) * [Filebeat] Add support for specifying AWS cred file (elastic#15656) * Add optional AWS shared_credential_file to all s3 input modules * Made AWS credential_profile_name optional for all s3 input modules Fixes elastic#15652 (cherry picked from commit a6a9c37) * update aws.asciidoc * update variables with default for 7.5 only Co-authored-by: Lee Hinman <[email protected]>
leweafan · Jan 28, 2020 · d81225b · d81225b
1 parent 2d3281d
commit d81225b
Show file tree

Hide file tree

Showing 11 changed files with 124 additions and 8 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -52,6 +52,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - cisco/asa fileset: Fix parsing of 302021 message code. {pull}14519[14519]
 - Fix filebeat azure dashboards, event category should be `Alert`. {pull}14668[14668]
 - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
+- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -35,15 +35,44 @@ Example config:
 ----
 - module: aws
   s3access:
-    enabled: true
-    var.queue_url: https://sqs.us-west-1.amazonaws.com/123/queue-name
-    var.credential_profile_name: fb-aws
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
+
+  elb:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
 
 AWS SQS queue url.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
 *`var.credential_profile_name`*::
 
 AWS credential profile name.

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -84,7 +84,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -93,7 +99,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
 #-------------------------------- Azure Module --------------------------------

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -5,7 +5,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -14,5 +20,11 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -30,15 +30,44 @@ Example config:
 ----
 - module: aws
   s3access:
-    enabled: true
-    var.queue_url: https://sqs.us-west-1.amazonaws.com/123/queue-name
-    var.credential_profile_name: fb-aws
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
+
+  elb:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
 
 AWS SQS queue url.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
 *`var.credential_profile_name`*::
 
 AWS credential profile name.
diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml
@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml
@@ -3,6 +3,10 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+    default: ~/.aws/credentials
+  - name: credential_profile_name
+    default: test
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

diff --git a/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json b/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json
@@ -179,4 +179,4 @@
         "source.ip": "77.227.156.41",
         "source.port": "46304"
     }
-]
+]
diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml
@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml
@@ -3,6 +3,9 @@ module_version: 1.0
 var:
   - name: input
     default: s3
-
+  - name: shared_credential_file
+    default: ~/.aws/credentials
+  - name: credential_profile_name
+    default: test
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled
@@ -8,7 +8,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -17,5 +23,11 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
-Original file line number
+Diff line change
@@ Expand Up / @@ -179,4 +179,4 @@ @@
             "source.ip": "77.227.156.41",
             "source.port": "46304"
         }
-    ]
+    ]