Merge remote-tracking branch 'upstream/master' into feature/beats-tes…

…ter-commit

* upstream/master:
  feat: package aliases for snapshots (elastic#21960)
  [DOC] Add firewall as possible troubleshooting issue (elastic#21743)
  [Filebeat] Add max_number_of_messages config parameter for S3 input (elastic#21993)
  [Elastic Agent] Fix missing elastic_agent event data  (elastic#21994)
  Document auditbeat system process module config (elastic#21766)
  Update links (elastic#22012)

.ci/packaging.groovy

@@ -210,10 +210,14 @@ def pushCIDockerImages(){
   }
 }
 
-def tagAndPush(name){
+def tagAndPush(beatName){
   def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
+    aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
     libbetaVer += "-SNAPSHOT"
+    aliasVersion += "-SNAPSHOT"
   }
 
   def tagName = "${libbetaVer}"
@@ -226,25 +230,37 @@ def tagAndPush(name){
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
   variants.each { variant ->
-    def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
-    def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
-    def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
-
-    def iterations = 0
-    retryWithSleep(retries: 3, seconds: 5, backoff: true) {
-      iterations++
-      def status = sh(label:'Change tag and push', script: """
-        docker tag ${oldName} ${newName}
-        docker push ${newName}
-        docker tag ${oldName} ${commitName}
-        docker push ${commitName}
-      """, returnStatus: true)
-
-      if ( status > 0 && iterations < 3) {
-        error('tag and push failed, retry')
-      } else if ( status > 0 ) {
-        log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
-      }
+    doTagAndPush(beatName, variant, libbetaVer, tagName)
+    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+    if (!isPR() && aliasVersion != "") {
+      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    }
+  }
+}
+
+/**
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+  def iterations = 0
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    iterations++
+    def status = sh(label: "Change tag and push ${targetName}", script: """
+      docker tag ${sourceName} ${targetName}
+      docker push ${targetName}
+    """, returnStatus: true)
+
+    if ( status > 0 && iterations < 3) {
+      error("tag and push failed for ${beatName}, retry")
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
     }
   }
 }

CHANGELOG.next.asciidoc

@@ -634,6 +634,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]
 - Adding support for FIPS in s3 input {pull}21446[21446]
 - Add SSL option to checkpoint module {pull}19560[19560]
+- Add max_number_of_messages config into s3 input. {pull}21993[21993]
 
 *Heartbeat*
 

libbeat/docs/getting-started.asciidoc

@@ -13,5 +13,5 @@ Each Beat is a separately installable product. To learn how to get started, see:
 * {winlogbeat-ref}/winlogbeat-installation-configuration.html[Winlogbeat]
 
 If you're planning to use the {metrics-app} or the {logs-app} in {kib},
-also see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].

libbeat/docs/howto/load-dashboards.asciidoc

@@ -15,8 +15,8 @@
 ifdef::has_solutions[]
 TIP: For deeper observability into your infrastructure, you can use the
 {metrics-app} and the {logs-app} in {kib}.
-For more details, see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+For more details, see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].
 endif::has_solutions[]
 
 {beatname_uc} comes packaged with example Kibana dashboards, visualizations,

libbeat/docs/overview.asciidoc

@@ -28,8 +28,8 @@ To get started, see <<getting-started>>.
 Want to get up and running quickly with infrastructure metrics monitoring and
 centralized log analytics?
 Try out the {metrics-app} and the {logs-app} in {kib}.
-For more details, see the {metrics-guide}[Metrics Monitoring Guide]
-and the {logs-guide}[Logs Monitoring Guide].
+For more details, see {observability-guide}/analyze-metrics.html[Analyze metrics]
+and {observability-guide}/monitor-logs.html[Monitor logs].
 
 [float]
 === Need to capture other kinds of data?

libbeat/docs/shared-faq.asciidoc

@@ -54,6 +54,27 @@ connect to the Lumberjack input plugin. To learn how to install and update
 plugins, see {logstash-ref}/working-with-plugins.html[Working with plugins].
 endif::[]
 
+ifndef::no-output-logstash[]
+[[publishing-ls-fails-connection-reset-by-peer]]
+=== Publishing to {ls} fails with "connection reset by peer" message
+
+{beatname_uc} requires a persistent TCP connection to {ls}. If a firewall interferes
+with the connection, you might see errors like this: 
+
+[source,shell]
+----------------------------------------------------------------------
+Failed to publish events caused by: write tcp ... write: connection reset by peer
+----------------------------------------------------------------------
+
+
+To solve the problem:
+
+* make sure the firewall is not closing connections between {beatname_uc} and {ls}, or
+* set the `ttl` value in the <<logstash-output,{ls} output>> to a value that's
+lower than the maximum time allowed by the firewall, and set `pipelining` to 0
+(pipelining cannot be enabled when `ttl` is used).
+endif::[]
+
 ifndef::no-output-logstash[]
 [[metadata-missing]]
 === @metadata is missing in {ls}

x-pack/auditbeat/docs/modules/system.asciidoc

@@ -97,7 +97,7 @@ This module also supports the
 <<module-standard-options-{modulename},standard configuration options>>
 described later.
 
-*`state.period`*:: The frequency at which the datasets send full state information.
+*`state.period`*:: The interval at which the datasets send full state information.
 This option can be overridden per dataset using `{dataset}.state.period`.
 
 *`user.detect_password_changes`*:: If the `user` dataset is configured and

x-pack/auditbeat/module/system/_meta/docs.asciidoc

@@ -90,7 +90,7 @@ This module also supports the
 <<module-standard-options-{modulename},standard configuration options>>
 described later.
 
-*`state.period`*:: The frequency at which the datasets send full state information.
+*`state.period`*:: The interval at which the datasets send full state information.
 This option can be overridden per dataset using `{dataset}.state.period`.
 
 *`user.detect_password_changes`*:: If the `user` dataset is configured and

x-pack/auditbeat/module/system/process/_meta/docs.asciidoc

@@ -2,10 +2,30 @@
 
 beta[]
 
-This is the `process` dataset of the system module.
+This is the `process` dataset of the system module. It generates an event when
+a process starts and stops.
 
 It is implemented for Linux, macOS (Darwin), and Windows.
 
+[float]
+=== Configuration options
+
+*`process.state.period`*:: The interval at which the dataset sends full state
+information. If set this will take precedence over `state.period`. The default
+value is `12h`.
+
+*`process.hash.max_file_size`*:: The maximum size of a file in bytes for which
+{beatname_uc} will compute hashes. Files larger than this size will not be
+hashed. The default value is 100 MiB. For convenience units can be specified as
+a suffix to the value. The supported units are `b` (default), `kib`, `kb`,
+`mib`, `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`.
+
+*`process.hash.hash_types`*:: A list of hash types to compute when the file
+changes. The supported hash types are `blake2b_256`, `blake2b_384`,
+`blake2b_512`, `md5`, `sha1`, `sha224`, `sha256`, `sha384`, `sha512`,
+`sha512_224`, `sha512_256`, `sha3_224`, `sha3_256`, `sha3_384`, `sha3_512`, and
+`xxh64`. The default value is `sha1`.
+
 [float]
 ==== Example dashboard
 

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -20,6 +20,7 @@
 - Use local temp instead of system one {pull}21883[21883]
 - Rename monitoring index from `elastic.agent` to `elastic_agent` {pull}21932[21932]
 - Fix issue with named pipes on Windows 7 {pull}21931[21931]
+- Fix missing elastic_agent event data {pull}21994[21994]
 
 ==== New features
 

x-pack/elastic-agent/pkg/agent/application/local_mode.go

@@ -100,7 +100,7 @@ func newLocal(
 		return nil, errors.New(err, "failed to initialize monitoring")
 	}
 
-	router, err := newRouter(log, streamFactory(localApplication.bgContext, cfg.Settings, localApplication.srv, reporter, monitor))
+	router, err := newRouter(log, streamFactory(localApplication.bgContext, agentInfo, cfg.Settings, localApplication.srv, reporter, monitor))
 	if err != nil {
 		return nil, errors.New(err, "fail to initialize pipeline router")
 	}

x-pack/elastic-agent/pkg/agent/application/managed_mode.go

@@ -154,7 +154,7 @@ func newManaged(
 		return nil, errors.New(err, "failed to initialize monitoring")
 	}
 
-	router, err := newRouter(log, streamFactory(managedApplication.bgContext, cfg.Settings, managedApplication.srv, combinedReporter, monitor))
+	router, err := newRouter(log, streamFactory(managedApplication.bgContext, agentInfo, cfg.Settings, managedApplication.srv, combinedReporter, monitor))
 	if err != nil {
 		return nil, errors.New(err, "fail to initialize pipeline router")
 	}

x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go

@@ -94,7 +94,6 @@ func getMonitoringRule(outputName string) *transpiler.RuleList {
 	return transpiler.NewRuleList(
 		transpiler.Copy(monitoringOutputSelector, outputKey),
 		transpiler.Rename(fmt.Sprintf("%s.%s", outputsKey, outputName), elasticsearchKey),
-		transpiler.InjectAgentInfo(),
 		transpiler.Filter(monitoringKey, programsKey, outputKey),
 	)
 }

x-pack/elastic-agent/pkg/agent/application/stream.go

@@ -7,6 +7,7 @@ package application
 import (
 	"context"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
@@ -40,10 +41,10 @@ func (b *operatorStream) Shutdown() {
 	b.configHandler.Shutdown()
 }
 
-func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) func(*logger.Logger, routingKey) (stream, error) {
+func streamFactory(ctx context.Context, agentInfo *info.AgentInfo, cfg *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) func(*logger.Logger, routingKey) (stream, error) {
 	return func(log *logger.Logger, id routingKey) (stream, error) {
 		// new operator per stream to isolate processes without using tags
-		operator, err := newOperator(ctx, log, id, cfg, srv, r, m)
+		operator, err := newOperator(ctx, log, agentInfo, id, cfg, srv, r, m)
 		if err != nil {
 			return nil, err
 		}
@@ -55,7 +56,7 @@ func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv *
 	}
 }
 
-func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) {
+func newOperator(ctx context.Context, log *logger.Logger, agentInfo *info.AgentInfo, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) {
 	fetcher := downloader.NewDownloader(log, config.DownloadConfig, false)
 	allowEmptyPgp, pgp := release.PGP()
 	verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp, false)
@@ -81,6 +82,7 @@ func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config
 	return operation.NewOperator(
 		ctx,
 		log,
+		agentInfo,
 		id,
 		config,
 		fetcher,

x-pack/elastic-agent/pkg/agent/operation/common_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/stateresolver"
@@ -48,6 +49,7 @@ func getTestOperator(t *testing.T, downloadPath string, installPath string, p *a
 	}
 
 	l := getLogger()
+	agentInfo, _ := info.NewAgentInfo()
 
 	fetcher := &DummyDownloader{}
 	verifier := &DummyVerifier{}
@@ -67,7 +69,7 @@ func getTestOperator(t *testing.T, downloadPath string, installPath string, p *a
 		t.Fatal(err)
 	}
 
-	operator, err := NewOperator(context.Background(), l, "p1", operatorCfg, fetcher, verifier, installer, uninstaller, stateResolver, srv, nil, noop.NewMonitor())
+	operator, err := NewOperator(context.Background(), l, agentInfo, "p1", operatorCfg, fetcher, verifier, installer, uninstaller, stateResolver, srv, nil, noop.NewMonitor())
 	if err != nil {
 		t.Fatal(err)
 	}

x-pack/elastic-agent/pkg/agent/operation/monitoring.go

@@ -206,6 +206,16 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i
 						},
 					},
 				},
+				{
+					"add_fields": map[string]interface{}{
+						"target": "elastic_agent",
+						"fields": map[string]interface{}{
+							"id":       o.agentInfo.AgentID(),
+							"version":  o.agentInfo.Version(),
+							"snapshot": o.agentInfo.Snapshot(),
+						},
+					},
+				},
 			},
 		},
 	}
@@ -240,6 +250,16 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i
 							},
 						},
 					},
+					{
+						"add_fields": map[string]interface{}{
+							"target": "elastic_agent",
+							"fields": map[string]interface{}{
+								"id":       o.agentInfo.AgentID(),
+								"version":  o.agentInfo.Version(),
+								"snapshot": o.agentInfo.Snapshot(),
+							},
+						},
+					},
 				},
 			})
 		}
@@ -290,6 +310,16 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string
 						},
 					},
 				},
+				{
+					"add_fields": map[string]interface{}{
+						"target": "elastic_agent",
+						"fields": map[string]interface{}{
+							"id":       o.agentInfo.AgentID(),
+							"version":  o.agentInfo.Version(),
+							"snapshot": o.agentInfo.Snapshot(),
+						},
+					},
+				},
 			},
 		})
 	}

x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/stateresolver"
@@ -112,6 +113,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M
 	}
 
 	l := getLogger()
+	agentInfo, _ := info.NewAgentInfo()
 
 	fetcher := &DummyDownloader{}
 	verifier := &DummyVerifier{}
@@ -128,7 +130,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M
 	}
 
 	ctx := context.Background()
-	operator, err := NewOperator(ctx, l, "p1", cfg, fetcher, verifier, installer, uninstaller, stateResolver, srv, nil, m)
+	operator, err := NewOperator(ctx, l, agentInfo, "p1", cfg, fetcher, verifier, installer, uninstaller, stateResolver, srv, nil, m)
 	if err != nil {
 		t.Fatal(err)
 	}

x-pack/elastic-agent/pkg/agent/operation/operator.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
@@ -43,6 +44,7 @@ type Operator struct {
 	bgContext     context.Context
 	pipelineID    string
 	logger        *logger.Logger
+	agentInfo     *info.AgentInfo
 	config        *configuration.SettingsConfig
 	handlers      map[string]handleFunc
 	stateResolver *stateresolver.StateResolver
@@ -66,6 +68,7 @@ type Operator struct {
 func NewOperator(
 	ctx context.Context,
 	logger *logger.Logger,
+	agentInfo *info.AgentInfo,
 	pipelineID string,
 	config *configuration.SettingsConfig,
 	fetcher download.Downloader,
@@ -85,6 +88,7 @@ func NewOperator(
 		config:        config,
 		pipelineID:    pipelineID,
 		logger:        logger,
+		agentInfo:     agentInfo,
 		downloader:    fetcher,
 		verifier:      verifier,
 		installer:     installer,