Merge remote-tracking branch 'upstream/master'

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.11" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.12", "7.11" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/apm-beats-update.groovy

@@ -86,10 +86,9 @@ pipeline {
               branch "v\\d?"
               tag "v\\d+\\.\\d+\\.\\d+*"
               allOf {
-                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+                expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() || isUserTrigger() }
                 changeRequest()
               }
-
             }
           }
           steps {
@@ -127,6 +126,7 @@ def beatsUpdate() {
         git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
 
         go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local
+        echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json
         make update
         git commit -a -m beats-update
 

.ci/packaging.groovy

@@ -247,13 +247,15 @@ pipeline {
             }
           }
         }
+        /*
         stage('Run E2E Tests for Packages'){
           agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           steps {
             runE2ETests()
           }
         }
+        */
       }
       post {
         success {

CHANGELOG.next.asciidoc

@@ -48,6 +48,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest PostgreSQL CSV logs. {pull}23334[23334]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
@@ -105,6 +106,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
 
 *Heartbeat*
+- Adds negative body match. {pull}20728[20728]
+- Refactor synthetics configuration to new syntax. {pull}23467[23467]
 
 *Journalbeat*
 
@@ -994,6 +997,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
 - Release MSSQL as GA {pull}23146[23146]
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
+- Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
 
 *Packetbeat*

NOTICE.txt

@@ -5891,11 +5891,11 @@ This Agreement is governed by the laws of the State of New York and the intellec
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.0.0-beta2.0.20210202203518-638aa2bb5271
+Version: v1.8.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.0.0-beta2.0.20210202203518-638aa2bb5271/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.8.0/LICENSE.txt:
 
 
                                  Apache License
@@ -7665,11 +7665,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.5.0
+Version: v1.6.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.5.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.6.0/LICENSE.txt:
 
 
                                  Apache License
@@ -8299,11 +8299,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.13.0
+Version: v0.14.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.13.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.14.0/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004

filebeat/docs/fields.asciidoc

@@ -112559,7 +112559,52 @@ The timestamp from the log line.
 *`postgresql.log.core_id`*::
 +
 --
-Core id
+
+deprecated:[8.0.0]
+
+Core id. (deprecated, there is no core_id in PostgreSQL logs, this is actually session_line_number).
+
+
+type: alias
+
+alias to: postgresql.log.session_line_number
+
+--
+
+*`postgresql.log.client_addr`*::
++
+--
+Host where the connection originated from.
+
+
+example: 127.0.0.1
+
+--
+
+*`postgresql.log.client_port`*::
++
+--
+Port where the connection originated from.
+
+
+example: 59700
+
+--
+
+*`postgresql.log.session_id`*::
++
+--
+PostgreSQL session.
+
+
+example: 5ff1dd98.22
+
+--
+
+*`postgresql.log.session_line_number`*::
++
+--
+Line number inside a session. (%l in `log_line_prefix`).
 
 
 type: long
@@ -112569,17 +112614,17 @@ type: long
 *`postgresql.log.database`*::
 +
 --
-Name of database
+Name of database.
 
 
-example: mydb
+example: postgres
 
 --
 
 *`postgresql.log.query`*::
 +
 --
-Query statement.
+Query statement. In the case of CSV parse, look at command_tag to get more context.
 
 
 example: SELECT * FROM users;
@@ -112589,7 +112634,7 @@ example: SELECT * FROM users;
 *`postgresql.log.query_step`*::
 +
 --
-Statement step when using extended query protocol (one of statement, parse, bind or execute)
+Statement step when using extended query protocol (one of statement, parse, bind or execute).
 
 
 example: parse
@@ -112606,30 +112651,153 @@ example: pdo_stmt_00000001
 
 --
 
-*`postgresql.log.error.code`*::
+*`postgresql.log.command_tag`*::
++
+--
+Type of session's current command. The complete list can be found at: src/include/tcop/cmdtaglist.h
+
+
+example: SELECT
+
+--
+
+*`postgresql.log.session_start_time`*::
++
+--
+Time when this session started.
+
+
+type: date
+
+--
+
+*`postgresql.log.virtual_transaction_id`*::
++
+--
+Backend local transaction id.
+
+
+--
+
+*`postgresql.log.transaction_id`*::
 +
 --
-Error code returned by Postgres (if any)
+The id of current transaction.
+
 
 type: long
 
 --
 
-*`postgresql.log.timezone`*::
+*`postgresql.log.sql_state_code`*::
 +
 --
+State code returned by Postgres (if any). See also https://www.postgresql.org/docs/current/errcodes-appendix.html
+
+
+type: keyword
+
+--
+
+*`postgresql.log.detail`*::
++
+--
+More information about the message, parameters in case of a parametrized query. e.g. 'Role \"user\" does not exist.', 'parameters: $1 = 42', etc.
+
+
+--
+
+*`postgresql.log.hint`*::
++
+--
+A possible solution to solve an error.
+
+
+--
+
+*`postgresql.log.internal_query`*::
++
+--
+Internal query that led to the error (if any).
+
+
+--
+
+*`postgresql.log.internal_query_pos`*::
++
+--
+Character count of the internal query (if any).
+
+
+type: long
+
+--
+
+*`postgresql.log.context`*::
++
+--
+Error context.
+
+
+--
+
+*`postgresql.log.query_pos`*::
++
+--
+Character count of the error position (if any).
+
+
+type: long
+
+--
+
+*`postgresql.log.location`*::
++
+--
+Location of the error in the PostgreSQL source code (if log_error_verbosity is set to verbose).
+
+
+--
+
+*`postgresql.log.application_name`*::
++
+--
+Name of the application of this event. It is defined by the client.
+
+
+--
+
+*`postgresql.log.backend_type`*::
++
+--
+Type of backend of this event. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. In addition, background workers registered by extensions may have additional types.
+
+
+example: client backend
+
+--
+
+*`postgresql.log.error.code`*::
++
+--
+
+deprecated:[8.0.0]
+
+Error code returned by Postgres (if any). Deprecated: errors can have letters. Use sql_state_code instead.
+
+
 type: alias
 
-alias to: event.timezone
+alias to: postgresql.log.sql_state_code
 
 --
 
-*`postgresql.log.thread_id`*::
+*`postgresql.log.timezone`*::
 +
 --
 type: alias
 
-alias to: process.pid
+alias to: event.timezone
 
 --
 
@@ -112645,8 +112813,13 @@ alias to: user.name
 *`postgresql.log.level`*::
 +
 --
+Valid values are DEBUG5, DEBUG4, DEBUG3, DEBUG2, DEBUG1, INFO, NOTICE, WARNING, ERROR, LOG, FATAL, and PANIC.
+
+
 type: alias
 
+example: LOG
+
 alias to: log.level
 
 --