elastic · kaiyan-sheng · Mar 12, 2020 · Feb 25, 2020 · Feb 25, 2020 · Feb 26, 2020
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
 - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
 - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529]
+- Add cloudwatch fileset in aws module. {issue}13716[13716] {pull}16579[16579]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1309,6 +1309,33 @@ type: keyword
 --
 Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3.
 
+type: keyword
+
+--
+
+[float]
+=== cloudwatch
+
+Fields for AWS CloudWatch logs.
+
+
+
+*`aws.cloudwatch.ip`*::
++
+--
+The internet address of the requester.
+
+
+type: ip
+
+--
+
+*`aws.cloudwatch.program_name`*::
++
+--
+The program name of the log entry.
+
+
 type: keyword
 
 --

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -31,7 +31,7 @@ Example config:
 [source,yaml]
 ----
 - module: aws
-  s3access:
+  cloudtrail:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -40,50 +40,42 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
 
-  elb:
+  cloudwatch:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  vpcflow:
+  elb:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  cloudtrail:
+  s3access:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
+  vpcflow:
+    enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
@@ -120,6 +112,16 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files
 that are delivered to the S3 bucket when Log File Integrity is turned
 on, it only reads the CloudTrail logs.
 
+[float]
+=== cloudwatch fileset
+
+Users can use Amazon CloudWatch Logs to monitor, store, and access log files
+from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources.
+Then export logs from log groups to Amazon S3 bucket which has SQS notification
+setup already. Right now, this fileset only parses EC2 logs into fields like `ip`
+and `program_name`. For logs from other services, this fileset will store them into
+`message` field.
+
 [float]
 === elb fileset
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -96,7 +96,33 @@ filebeat.modules:
 
 #--------------------------------- AWS Module ---------------------------------
 - module: aws
-  s3access:
+  cloudtrail:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  cloudwatch:
     enabled: false
 
     # AWS SQS queue url
@@ -148,7 +174,7 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  vpcflow:
+  s3access:
     enabled: false
 
     # AWS SQS queue url
@@ -174,7 +200,7 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  cloudtrail:
+  vpcflow:
     enabled: false
 
     # AWS SQS queue url

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -1,5 +1,31 @@
 - module: aws
-  s3access:
+  cloudtrail:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  cloudwatch:
     enabled: false
 
     # AWS SQS queue url
@@ -51,7 +77,7 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  vpcflow:
+  s3access:
     enabled: false
 
     # AWS SQS queue url
@@ -77,7 +103,7 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
-  cloudtrail:
+  vpcflow:
     enabled: false
 
     # AWS SQS queue url

diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -26,7 +26,7 @@ Example config:
 [source,yaml]
 ----
 - module: aws
-  s3access:
+  cloudtrail:
     enabled: false
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
@@ -35,50 +35,42 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
 
-  elb:
+  cloudwatch:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  vpcflow:
+  elb:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
-
-  cloudtrail:
+  s3access:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
+  vpcflow:
+    enabled: false
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
@@ -115,6 +107,16 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files
 that are delivered to the S3 bucket when Log File Integrity is turned
 on, it only reads the CloudTrail logs.
 
+[float]
+=== cloudwatch fileset
+
+Users can use Amazon CloudWatch Logs to monitor, store, and access log files
+from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources.
+Then export logs from log groups to Amazon S3 bucket which has SQS notification
+setup already. Right now, this fileset only parses EC2 logs into fields like `ip`
+and `program_name`. For logs from other services, this fileset will store them into
+`message` field.
+
 [float]
 === elb fileset
 

diff --git a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml
@@ -0,0 +1,15 @@
+- name: cloudwatch
+  type: group
+  release: beta
+  default_field: false
+  description: >
+    Fields for AWS CloudWatch logs.
+  fields:
+    - name: ip
+      type: ip
+      description: >
+        The internet address of the requester.
+    - name: program_name
+      type: keyword
+      description: >
+        The program name of the log entry.
diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml b/x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml
@@ -0,0 +1,22 @@
+type: s3
+queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
+credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
+
+{{ if .visibility_timeout }}
+visibility_timeout: {{ .visibility_timeout }}
+{{ end }}
+
+{{ if .api_timeout }}
+api_timeout: {{ .api_timeout }}
+{{ end }}
+
+{{ if .endpoint }}
+endpoint: {{ .endpoint }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml
@@ -0,0 +1,6 @@
+type: log
+paths:
+  {{ range $i, $path := .paths }}
+  - {{$path}}
+    {{ end }}
+exclude_files: [".gz$"]