diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e1aeec8868..beb7f3f693d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -558,6 +558,38 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] +- Update PanOS module to parse Global Protect & User ID logs. {issue}24722[24722] {issue}24724[24724] {pull}24927[24927] +- Add HMAC signature validation support for http_endpoint input. {pull}24918[24918] +- Add new grok pattern for iptables module for Ubiquiti UDM {issue}25615[25615] {pull}25616[25616] +- Add multiline support to aws-s3 input. {issue}25249[25249] {pull}25710[25710] {pull}25873[25873] +- Add monitoring metrics to the `aws-s3` input. {pull}25711[25711] +- Added `network.direction` fields to Zeek and Suricata modules using the `add_network_direction` processor {pull}24620[24620] +- Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772] +- In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776] +- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841] +- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686] +- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774] +- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] +- Add log_group_name_prefix config into aws-cloudwatch input. {pull}26187[26187] +- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168] +- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764] +- Make `filestream` input GA. {pull}26127[26127] +- Add new `parser` to `filestream` input: `container`. {pull}26115[26115] +- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564] +- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279] +- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273] +- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267] +- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267] +- RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293] +- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835] +- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158] +- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818] +- Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] +- Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] +- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441] +- Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481] +- Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816] +- Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524] *Heartbeat* diff --git a/README.md b/README.md index e76f80a695e..8050346f791 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.x/badge/icon)](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.x/) +[![Build Status](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.13/badge/icon)](https://beats-ci.elastic.co/job/Beats/job/beats/job/7.13/) [![GoReportCard](http://goreportcard.com/badge/elastic/beats)](http://goreportcard.com/report/elastic/beats) [![codecov.io](https://codecov.io/github/elastic/beats/coverage.svg?branch=master)](https://codecov.io/github/elastic/beats?branch=master) diff --git a/deploy/kubernetes/auditbeat-kubernetes.yaml b/deploy/kubernetes/auditbeat-kubernetes.yaml index 5498ed21a83..77b7303e53f 100644 --- a/deploy/kubernetes/auditbeat-kubernetes.yaml +++ b/deploy/kubernetes/auditbeat-kubernetes.yaml @@ -109,7 +109,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: auditbeat - image: docker.elastic.co/beats/auditbeat:7.11.0 + image: docker.elastic.co/beats/auditbeat:7.13.0 args: [ "-c", "/etc/auditbeat.yml", "-e", diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml index 78a9552a78e..71b0b0e5fe7 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml @@ -458,7 +458,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.11.0 + image: docker.elastic.co/beats/elastic-agent:7.13.0 args: [ "-c", "/etc/agent.yml", "-e", diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index a252863fb2a..715cf7c368e 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -64,7 +64,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: filebeat - image: docker.elastic.co/beats/filebeat:7.11.0 + image: docker.elastic.co/beats/filebeat:7.13.0 args: [ "-c", "/etc/filebeat.yml", "-e", diff --git a/deploy/kubernetes/heartbeat-kubernetes.yaml b/deploy/kubernetes/heartbeat-kubernetes.yaml index af01d6fed16..bad444c75b0 100644 --- a/deploy/kubernetes/heartbeat-kubernetes.yaml +++ b/deploy/kubernetes/heartbeat-kubernetes.yaml @@ -74,7 +74,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: heartbeat - image: docker.elastic.co/beats/heartbeat:7.11.0 + image: docker.elastic.co/beats/heartbeat:7.13.0 args: [ "-c", "/etc/heartbeat.yml", "-e", diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 8315cb0d739..dd31be41425 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -147,7 +147,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: metricbeat - image: docker.elastic.co/beats/metricbeat:7.11.0 + image: docker.elastic.co/beats/metricbeat:7.13.0 args: [ "-c", "/etc/metricbeat.yml", "-e", diff --git a/filebeat/docs/modules/fortinet.asciidoc b/filebeat/docs/modules/fortinet.asciidoc index 78c8825bdd1..eb6e6a3f014 100644 --- a/filebeat/docs/modules/fortinet.asciidoc +++ b/filebeat/docs/modules/fortinet.asciidoc @@ -27,7 +27,7 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against FortiOS version 6.0.x and 6.2.x. +This module has been tested against FortiOS version 6.0.x and 6.2.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] @@ -51,6 +51,8 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + *`var.input`*:: The input to use, can be either the value `tcp`, `udp` or `file`. diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 738d7d5c145..e126a7ca0c7 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,4 +1,4 @@ -:stack-version: 7.11.0 +:stack-version: 7.13.0 :doc-branch: 7.x :go-version: 1.15.10 :release-state: unreleased diff --git a/testing/environments/snapshot-oss.yml b/testing/environments/snapshot-oss.yml index 6691044e03f..19758bf474c 100644 --- a/testing/environments/snapshot-oss.yml +++ b/testing/environments/snapshot-oss.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.12.0-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.13.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -15,7 +15,7 @@ services: - "http.host=0.0.0.0" logstash: - image: docker.elastic.co/logstash/logstash-oss:7.12.0-SNAPSHOT + image: docker.elastic.co/logstash/logstash-oss:7.13.0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -25,7 +25,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana-oss:7.12.0-SNAPSHOT + image: docker.elastic.co/kibana/kibana-oss:7.13.0-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status | grep -q 'Looking good'"] retries: 600 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index df9588dac50..f48c558a5d0 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2223,8 +2223,6 @@ filebeat.modules: # The interval to poll the API for updates var.interval: 5m -======= - anomali: enabled: true diff --git a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc index acf35ecacc9..b504fc1cbf4 100644 --- a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc @@ -22,7 +22,7 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against FortiOS version 6.0.x and 6.2.x. +This module has been tested against FortiOS version 6.0.x and 6.2.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] @@ -46,6 +46,8 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + *`var.input`*:: The input to use, can be either the value `tcp`, `udp` or `file`. diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index f83d8bfe67d..525064cbacc 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -25,6 +25,7 @@ tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: + - add_locale: ~ - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index c103fd14700..3a8ed0a8b26 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -47,12 +47,12 @@ processors: ignore_empty_value: true - set: field: _temp.time - value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null" + value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{event.timezone}}" + if: "ctx.event?.timezone != null" - set: field: _temp.time value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}" - if: "ctx.fortinet?.firewall?.tz == null" + if: "ctx.event?.timezone == null" - date: field: _temp.time target_field: "@timestamp" @@ -61,8 +61,8 @@ processors: - yyyy-MM-dd HH:mm:ss Z - yyyy-MM-dd HH:mm:ss z - ISO8601 - timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null" + timezone: "{{event.timezone}}" + if: "ctx.event?.timezone != null" - date: field: _temp.time target_field: "@timestamp" @@ -71,7 +71,7 @@ processors: - yyyy-MM-dd HH:mm:ss Z - yyyy-MM-dd HH:mm:ss z - ISO8601 - if: "ctx.fortinet?.firewall?.tz == null" + if: "ctx.event?.timezone == null" - gsub: field: fortinet.firewall.eventtime pattern: "\\d{6}$" @@ -82,27 +82,27 @@ processors: target_field: event.start formats: - UNIX_MS - timezone: "{{fortinet.firewall.tz}}" - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + timezone: "{{event.timezone}}" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - timezone: "{{fortinet.firewall.tz}}" - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + timezone: "{{event.timezone}}" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - script: lang: painless source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 172748796d1..9302e52f727 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -76,7 +76,7 @@ "url.path": "/config/" }, { - "@timestamp": "2020-04-23T01:16:08.000Z", + "@timestamp": "2020-04-23T01:16:08.000-02:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", "destination.bytes": 0, @@ -97,7 +97,8 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", - "event.start": "2020-06-24T01:16:08.000Z", + "event.start": "2020-06-23T23:16:08.000-02:00", + "event.timezone": "-02:00", "event.type": [ "connection", "end" @@ -1905,7 +1906,7 @@ ] }, { - "@timestamp": "2019-05-15T18:03:36.000Z", + "@timestamp": "2019-05-15T18:03:36.000-02:00", "destination.as.number": 41690, "destination.as.organization.name": "Dailymotion S.A.", "destination.geo.continent_name": "Europe", @@ -1924,7 +1925,8 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", - "event.start": "2019-05-16T01:03:35.000Z", + "event.start": "2019-05-15T23:03:35.000-02:00", + "event.timezone": "-02:00", "event.type": [ "allowed" ], @@ -1973,7 +1975,7 @@ "url.path": "/" }, { - "@timestamp": "2020-11-02T08:11:38.000Z", + "@timestamp": "2020-11-02T08:11:38.000-02:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", "destination.geo.continent_name": "North America", @@ -1991,6 +1993,7 @@ "event.kind": "event", "event.module": "fortinet", "event.outcome": "success", + "event.timezone": "-02:00", "event.type": [ "connection" ], diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml index f754cae2144..9a78adca0ae 100644 --- a/x-pack/filebeat/module/threatintel/_meta/config.yml +++ b/x-pack/filebeat/module/threatintel/_meta/config.yml @@ -91,8 +91,6 @@ # The interval to poll the API for updates var.interval: 5m -======= - anomali: enabled: true diff --git a/x-pack/filebeat/modules.d/threatintel.yml.disabled b/x-pack/filebeat/modules.d/threatintel.yml.disabled index e64e05a81f7..ad571b10e9c 100644 --- a/x-pack/filebeat/modules.d/threatintel.yml.disabled +++ b/x-pack/filebeat/modules.d/threatintel.yml.disabled @@ -94,8 +94,6 @@ # The interval to poll the API for updates var.interval: 5m -======= - anomali: enabled: true