Cherry-pick #26524 to 7.x: [filebeat][fortinet] Use default add_locale for fortinet.firewall

CHANGELOG.next.asciidoc

@@ -558,6 +558,38 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041]
 - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
 - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
+- Update PanOS module to parse Global Protect & User ID logs. {issue}24722[24722] {issue}24724[24724] {pull}24927[24927]
+- Add HMAC signature validation support for http_endpoint input. {pull}24918[24918]
+- Add new grok pattern for iptables module for Ubiquiti UDM {issue}25615[25615] {pull}25616[25616]
+- Add multiline support to aws-s3 input. {issue}25249[25249] {pull}25710[25710] {pull}25873[25873]
+- Add monitoring metrics to the `aws-s3` input. {pull}25711[25711]
+- Added `network.direction` fields to Zeek and Suricata modules using the `add_network_direction` processor {pull}24620[24620]
+- Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772]
+- In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
+- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
+- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
+- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
+- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
+- Add log_group_name_prefix config into aws-cloudwatch input. {pull}26187[26187]
+- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
+- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
+- Make `filestream` input GA. {pull}26127[26127]
+- Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
+- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279]
+- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
+- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
+- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
+- RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
+- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
+- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158]
+- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
+- Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
+- Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
+- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
+- Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
+- Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
+- Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
 
 *Heartbeat*
 

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.x/badge/icon)](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.x/)
+[![Build Status](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.13/badge/icon)](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.13/)
 [![GoReportCard](http://goreportcard.com/badge/elastic/beats)](http://goreportcard.com/report/elastic/beats)
 [![codecov.io](https://codecov.io/github/elastic/beats/coverage.svg?branch=master)](https://codecov.io/github/elastic/beats?branch=master)
 

deploy/kubernetes/auditbeat-kubernetes.yaml

@@ -109,7 +109,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: auditbeat
-        image: docker.elastic.co/beats/auditbeat:7.11.0
+        image: docker.elastic.co/beats/auditbeat:7.13.0
         args: [
           "-c", "/etc/auditbeat.yml",
           "-e",

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -458,7 +458,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: docker.elastic.co/beats/elastic-agent:7.11.0
+          image: docker.elastic.co/beats/elastic-agent:7.13.0
           args: [
             "-c", "/etc/agent.yml",
             "-e",

deploy/kubernetes/filebeat-kubernetes.yaml

@@ -64,7 +64,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: filebeat
-        image: docker.elastic.co/beats/filebeat:7.11.0
+        image: docker.elastic.co/beats/filebeat:7.13.0
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",

deploy/kubernetes/heartbeat-kubernetes.yaml

@@ -74,7 +74,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: heartbeat
-        image: docker.elastic.co/beats/heartbeat:7.11.0
+        image: docker.elastic.co/beats/heartbeat:7.13.0
         args: [
           "-c", "/etc/heartbeat.yml",
           "-e",

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -147,7 +147,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: metricbeat
-        image: docker.elastic.co/beats/metricbeat:7.11.0
+        image: docker.elastic.co/beats/metricbeat:7.13.0
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",

filebeat/docs/modules/fortinet.asciidoc

@@ -27,7 +27,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-This module has been tested against FortiOS version 6.0.x and 6.2.x. 
+This module has been tested against FortiOS version 6.0.x and 6.2.x.
 Versions above this are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]
@@ -51,6 +51,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/timezone-support.asciidoc[]
+
 *`var.input`*::
 
 The input to use, can be either the value `tcp`, `udp` or `file`.

libbeat/docs/version.asciidoc

@@ -1,4 +1,4 @@
-:stack-version: 7.11.0
+:stack-version: 7.13.0
 :doc-branch: 7.x
 :go-version: 1.15.10
 :release-state: unreleased

testing/environments/snapshot-oss.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.12.0-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.13.0-SNAPSHOT
     healthcheck:
       test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"]
       retries: 300
@@ -15,7 +15,7 @@ services:
     - "http.host=0.0.0.0"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash-oss:7.12.0-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash-oss:7.13.0-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -25,7 +25,7 @@ services:
     - ./docker/logstash/pki:/etc/pki:ro
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:7.12.0-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana-oss:7.13.0-SNAPSHOT
     healthcheck:
       test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q 'Looking good'"]
       retries: 600

x-pack/filebeat/filebeat.reference.yml

@@ -2223,8 +2223,6 @@ filebeat.modules:
     # The interval to poll the API for updates
     var.interval: 5m
 
-=======
-
   anomali:
     enabled: true
 

x-pack/filebeat/module/fortinet/_meta/docs.asciidoc

@@ -22,7 +22,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-This module has been tested against FortiOS version 6.0.x and 6.2.x. 
+This module has been tested against FortiOS version 6.0.x and 6.2.x.
 Versions above this are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]
@@ -46,6 +46,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/timezone-support.asciidoc[]
+
 *`var.input`*::
 
 The input to use, can be either the value `tcp`, `udp` or `file`.

x-pack/filebeat/module/fortinet/firewall/config/firewall.yml

@@ -25,6 +25,7 @@ tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
+  - add_locale: ~
   - add_fields:
       target: ''
       fields:

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -47,12 +47,12 @@ processors:
     ignore_empty_value: true
 - set:
     field: _temp.time
-    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null"
+    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{event.timezone}}"
+    if: "ctx.event?.timezone != null"
 - set:
     field: _temp.time
     value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}"
-    if: "ctx.fortinet?.firewall?.tz == null"
+    if: "ctx.event?.timezone == null"
 - date:
     field: _temp.time
     target_field: "@timestamp"
@@ -61,8 +61,8 @@ processors:
     - yyyy-MM-dd HH:mm:ss Z
     - yyyy-MM-dd HH:mm:ss z
     - ISO8601
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null"
+    timezone: "{{event.timezone}}"
+    if: "ctx.event?.timezone != null"
 - date:
     field: _temp.time
     target_field: "@timestamp"
@@ -71,7 +71,7 @@ processors:
     - yyyy-MM-dd HH:mm:ss Z
     - yyyy-MM-dd HH:mm:ss z
     - ISO8601
-    if: "ctx.fortinet?.firewall?.tz == null"
+    if: "ctx.event?.timezone == null"
 - gsub:
     field: fortinet.firewall.eventtime
     pattern: "\\d{6}$"
@@ -82,27 +82,27 @@ processors:
     target_field: event.start
     formats:
     - UNIX_MS
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    timezone: "{{event.timezone}}"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    timezone: "{{event.timezone}}"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - script:
     lang: painless
     source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"

x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json

@@ -76,7 +76,7 @@
         "url.path": "/config/"
     },
     {
-        "@timestamp": "2020-04-23T01:16:08.000Z",
+        "@timestamp": "2020-04-23T01:16:08.000-02:00",
         "destination.as.number": 15169,
         "destination.as.organization.name": "Google LLC",
         "destination.bytes": 0,
@@ -97,7 +97,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2020-06-24T01:16:08.000Z",
+        "event.start": "2020-06-23T23:16:08.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "end"
@@ -1905,7 +1906,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-15T18:03:36.000Z",
+        "@timestamp": "2019-05-15T18:03:36.000-02:00",
         "destination.as.number": 41690,
         "destination.as.organization.name": "Dailymotion S.A.",
         "destination.geo.continent_name": "Europe",
@@ -1924,7 +1925,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-16T01:03:35.000Z",
+        "event.start": "2019-05-15T23:03:35.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "allowed"
         ],
@@ -1973,7 +1975,7 @@
         "url.path": "/"
     },
     {
-        "@timestamp": "2020-11-02T08:11:38.000Z",
+        "@timestamp": "2020-11-02T08:11:38.000-02:00",
         "destination.as.number": 15169,
         "destination.as.organization.name": "Google LLC",
         "destination.geo.continent_name": "North America",
@@ -1991,6 +1993,7 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection"
         ],

x-pack/filebeat/module/threatintel/_meta/config.yml

@@ -91,8 +91,6 @@
     # The interval to poll the API for updates
     var.interval: 5m
 
-=======
-
   anomali:
     enabled: true
 

x-pack/filebeat/modules.d/threatintel.yml.disabled

@@ -94,8 +94,6 @@
     # The interval to poll the API for updates
     var.interval: 5m
 
-=======
-
   anomali:
     enabled: true