Cherry-pick #16579 to 7.x: [Filebeat] Add cloudwatch fileset in aws m…

…odule (#16978) * [Filebeat] Add cloudwatch fileset in aws module (#16579) * Add cloudwatch fileset in aws module (cherry picked from commit ad4597c)
elastic · Mar 17, 2020 · cd73cce · cd73cce
1 parent 02d3c70
commit cd73cce
Show file tree

Hide file tree

Showing 22 changed files with 517 additions and 78 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201]
 - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
 - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
+- Add cloudwatch fileset and ec2 fileset in aws module. {issue}13716[13716] {pull}16579[16579]
 - Improve ECS categorization field mappings in kibana module. {issue}16168[16168] {pull}16652[16652]
 - Add `cloudfoundry` input to send events from Cloud Foundry. {pull}16586[16586]
 - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529]

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1309,6 +1309,29 @@ type: keyword
 --
 Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3.
 
+type: keyword
+
+--
+
+[float]
+=== cloudwatch
+
+Fields for AWS CloudWatch logs.
+
+
+[float]
+=== ec2
+
+Fields for AWS EC2 logs in CloudWatch.
+
+
+
+*`aws.ec2.ip_address`*::
++
+--
+The internet address of the requester.
+
+
 type: keyword
 
 --

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -33,7 +33,7 @@ Example config:
 [source,yaml]
 ----
 - module: aws
-  s3access:
+  cloudtrail:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -42,50 +42,51 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
 
-  elb:
+  cloudwatch:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  vpcflow:
+  ec2:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  cloudtrail:
+  elb:
     enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # AWS SQS queue url
+  s3access:
+    enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
+  vpcflow:
+    enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
@@ -122,6 +123,22 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files
 that are delivered to the S3 bucket when Log File Integrity is turned
 on, it only reads the CloudTrail logs.
 
+[float]
+=== cloudwatch fileset
+
+Users can use Amazon CloudWatch Logs to monitor, store, and access log files
+from different sources. Export logs from log groups to an Amazon S3 bucket which
+has SQS notification setup already. This fileset will parse these logs into
+`timestamp` and `message` field.
+
+[float]
+=== ec2 fileset
+
+This fileset is specifically for EC2 logs stored in AWS CloudWatch. Export logs
+from log groups to Amazon S3 bucket which has SQS notification setup already.
+With this fileset, EC2 logs will be parsed into fields like  `ip`
+and `program_name`. For logs from other services, please use `cloudwatch` fileset.
+
 [float]
 === elb fileset
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -96,7 +96,33 @@ filebeat.modules:
 
 #--------------------------------- AWS Module ---------------------------------
 - module: aws
-  s3access:
+  cloudtrail:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  cloudwatch:
     enabled: false
 
     # AWS SQS queue url
@@ -148,7 +174,7 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  vpcflow:
+  s3access:
     enabled: false
 
     # AWS SQS queue url
@@ -174,7 +200,7 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  cloudtrail:
+  vpcflow:
     enabled: false
 
     # AWS SQS queue url

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -1,5 +1,31 @@
 - module: aws
-  s3access:
+  cloudtrail:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  cloudwatch:
     enabled: false
 
     # AWS SQS queue url
@@ -51,7 +77,7 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  vpcflow:
+  s3access:
     enabled: false
 
     # AWS SQS queue url
@@ -77,7 +103,7 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  cloudtrail:
+  vpcflow:
     enabled: false
 
     # AWS SQS queue url

diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -28,7 +28,7 @@ Example config:
 [source,yaml]
 ----
 - module: aws
-  s3access:
+  cloudtrail:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -37,50 +37,51 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
 
-  elb:
+  cloudwatch:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  vpcflow:
+  ec2:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  cloudtrail:
+  elb:
     enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # AWS SQS queue url
+  s3access:
+    enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
+  vpcflow:
+    enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
@@ -117,6 +118,22 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files
 that are delivered to the S3 bucket when Log File Integrity is turned
 on, it only reads the CloudTrail logs.
 
+[float]
+=== cloudwatch fileset
+
+Users can use Amazon CloudWatch Logs to monitor, store, and access log files
+from different sources. Export logs from log groups to an Amazon S3 bucket which
+has SQS notification setup already. This fileset will parse these logs into
+`timestamp` and `message` field.
+
+[float]
+=== ec2 fileset
+
+This fileset is specifically for EC2 logs stored in AWS CloudWatch. Export logs
+from log groups to Amazon S3 bucket which has SQS notification setup already.
+With this fileset, EC2 logs will be parsed into fields like  `ip`
+and `program_name`. For logs from other services, please use `cloudwatch` fileset.
+
 [float]
 === elb fileset
 

diff --git a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml
@@ -0,0 +1,7 @@
+- name: cloudwatch
+  type: group
+  release: beta
+  default_field: false
+  description: >
+    Fields for AWS CloudWatch logs.
+  fields: