From 531bf81560b54c52e01fb43e0cbf7cb41408e854 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 8 Apr 2021 10:38:36 +0300 Subject: [PATCH 01/12] Update k8s manifests to use proper roles' scope for leaderelection (#24958) --- deploy/kubernetes/metricbeat-kubernetes.yaml | 34 +++++++++++++++---- .../metricbeat/metricbeat-role-binding.yaml | 14 ++++++++ .../metricbeat/metricbeat-role.yaml | 20 +++++++---- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index ce685aa09753..952274c5420c 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -231,6 +231,20 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: metricbeat @@ -270,12 +284,20 @@ rules: - "/metrics" verbs: - get -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] --- apiVersion: v1 kind: ServiceAccount diff --git a/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml index 3f6f7b62439f..a3a4438e068d 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role-binding.yaml @@ -10,3 +10,17 @@ roleRef: kind: ClusterRole name: metricbeat apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml index 0eb2e89c7bd1..74a97e1d38d0 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -38,9 +38,17 @@ rules: - "/metrics" verbs: - get -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] From 2494ae509b1036d39bd1bbb4ec751c50347c9aff Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 8 Apr 2021 11:14:33 +0300 Subject: [PATCH 02/12] Add kubernetes_leaderelection provider (#24913) --- .../elastic-agent-standalone-kubernetes.yml | 32 ++++ .../elastic-agent-standalone-daemonset.yaml | 4 + ...elastic-agent-standalone-role-binding.yaml | 14 ++ .../elastic-agent-standalone-role.yaml | 14 ++ x-pack/elastic-agent/pkg/agent/cmd/include.go | 1 + .../kubernetesleaderelection/config.go | 20 +++ .../kubernetes_leaderelection.go | 148 ++++++++++++++++++ 7 files changed, 233 insertions(+) create mode 100644 x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/config.go create mode 100644 x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index 5d83516f686c..6da1bd9332c6 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -36,6 +36,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name securityContext: runAsUser: 0 resources: @@ -650,6 +654,34 @@ rules: verbs: - get --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: kube-system + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- apiVersion: v1 kind: ServiceAccount metadata: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index e97e07439263..f23e2cb1534b 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -38,6 +38,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name securityContext: runAsUser: 0 resources: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml index b352b2901d0d..f053b2463371 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml @@ -10,3 +10,17 @@ roleRef: kind: ClusterRole name: elastic-agent apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: kube-system + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: Role + name: elastic-agent + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml index 13b3554b83cc..11089d0cc7af 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml @@ -38,3 +38,17 @@ rules: - "/metrics" verbs: - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] diff --git a/x-pack/elastic-agent/pkg/agent/cmd/include.go b/x-pack/elastic-agent/pkg/agent/cmd/include.go index 5bc763c6df06..f28b5cb11b50 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/include.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/include.go @@ -11,6 +11,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/env" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/host" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/kubernetes" + _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/kubernetessecrets" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/local" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/localdynamic" diff --git a/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/config.go b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/config.go new file mode 100644 index 000000000000..a7f71cc32b5d --- /dev/null +++ b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// TODO review the need for this +// +build linux darwin windows + +package kubernetesleaderelection + +// Config for kubernetes_leaderelection provider +type Config struct { + KubeConfig string `config:"kube_config"` + // Name of the leaderelection lease + LeaderLease string `config:"leader_lease"` +} + +// InitDefaults initializes the default values for the config. +func (c *Config) InitDefaults() { + c.LeaderLease = "elastic-agent-cluster-leader" +} diff --git a/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go new file mode 100644 index 000000000000..acb5e732b824 --- /dev/null +++ b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package kubernetesleaderelection + +import ( + "context" + "os" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + + "github.com/elastic/beats/v7/libbeat/common/kubernetes" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" + corecomp "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/composable" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" +) + +func init() { + composable.Providers.AddContextProvider("kubernetes_leaderelection", ContextProviderBuilder) +} + +type contextProvider struct { + logger *logger.Logger + config *Config + comm corecomp.ContextProviderComm + leaderElection *leaderelection.LeaderElectionConfig + cancelLeaderElection context.CancelFunc +} + +// ContextProviderBuilder builds the provider. +func ContextProviderBuilder(logger *logger.Logger, c *config.Config) (corecomp.ContextProvider, error) { + var cfg Config + if c == nil { + c = config.New() + } + err := c.Unpack(&cfg) + if err != nil { + return nil, errors.New(err, "failed to unpack configuration") + } + return &contextProvider{logger, &cfg, nil, nil, nil}, nil +} + +// Run runs the leaderelection provider. +func (p *contextProvider) Run(comm corecomp.ContextProviderComm) error { + client, err := kubernetes.GetKubernetesClient(p.config.KubeConfig) + if err != nil { + // info only; return nil (do nothing) + p.logger.Debugf("Kubernetes leaderelection provider skipped, unable to connect: %s", err) + return nil + } + + agentInfo, err := info.NewAgentInfo() + if err != nil { + return err + } + var id string + podName, found := os.LookupEnv("POD_NAME") + if found { + id = "elastic-agent-leader-" + podName + } else { + id = "elastic-agent-leader-" + agentInfo.AgentID() + } + + ns, err := kubernetes.InClusterNamespace() + if err != nil { + ns = "default" + } + lease := metav1.ObjectMeta{ + Name: p.config.LeaderLease, + Namespace: ns, + } + metaUID := lease.GetObjectMeta().GetUID() + p.leaderElection = &leaderelection.LeaderElectionConfig{ + Lock: &resourcelock.LeaseLock{ + LeaseMeta: lease, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: id, + }, + }, + ReleaseOnCancel: true, + LeaseDuration: 15 * time.Second, + RenewDeadline: 10 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + p.logger.Debugf("leader election lock GAINED, id %v", id) + p.startLeading(string(metaUID)) + }, + OnStoppedLeading: func() { + p.logger.Debugf("leader election lock LOST, id %v", id) + p.stopLeading(string(metaUID)) + }, + }, + } + ctx, cancel := context.WithCancel(context.TODO()) + p.cancelLeaderElection = cancel + p.comm = comm + p.startLeaderElector(ctx) + + return nil +} + +// startLeaderElector starts a Leader Elector in the background with the provided config +func (p *contextProvider) startLeaderElector(ctx context.Context) { + le, err := leaderelection.NewLeaderElector(*p.leaderElection) + if err != nil { + p.logger.Errorf("error while creating Leader Elector: %v", err) + } + p.logger.Debugf("Starting Leader Elector") + go le.Run(ctx) +} + +func (p *contextProvider) startLeading(metaUID string) { + mapping := map[string]interface{}{ + "leader": true, + } + + err := p.comm.Set(mapping) + if err != nil { + p.logger.Errorf("Failed updating leaderelection status to leader TRUE: %s", err) + } +} + +func (p *contextProvider) stopLeading(metaUID string) { + mapping := map[string]interface{}{ + "leader": false, + } + + err := p.comm.Set(mapping) + if err != nil { + p.logger.Errorf("Failed updating leaderelection status to leader FALSE: %s", err) + } +} + +// Stop signals the stop channel to force the leader election loop routine to stop. +func (p *contextProvider) Stop() { + if p.cancelLeaderElection != nil { + p.cancelLeaderElection() + } +} From b0f293e00accab7565c48d7892afdc47fca03007 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 8 Apr 2021 11:57:09 +0300 Subject: [PATCH 03/12] Add changelog entry for leaderelection on Agent (#24985) Signed-off-by: chrismark --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 246ad6b8bc74..213454408136 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -83,3 +83,4 @@ - Add k8s secrets provider for Agent {pull}24789[24789] - Add STATE_PATH, CONFIG_PATH, LOGS_PATH to Elastic Agent docker image {pull}24817[24817] - Add status subcommand {pull}24856[24856] +- Add leader_election provider for k8s {pull}24267[24267] From 974f2555477b37c224706bbd0c65538df9875016 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 8 Apr 2021 11:27:19 -0400 Subject: [PATCH 04/12] [Elastic Agent] Adjust to Fleet Server connection information from Kibana in configuration (#24713) * Adjust fleet.kibana.* to just fleet.* * Switch from fleet.kibana.* to fleet.* for connection. * Add changelog entry. * Add check to ensure no possible panic in mapDict. --- x-pack/elastic-agent/.gitignore | 3 + x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + .../pkg/agent/application/managed_mode.go | 4 +- .../handlers/handler_action_policy_change.go | 43 +++++++------ .../emitter/modifiers/fleet_decorator.go | 43 ++++++++----- .../elastic-agent/pkg/agent/cmd/enroll_cmd.go | 24 +++---- .../pkg/agent/cmd/enroll_cmd_test.go | 18 +++--- .../pkg/agent/configuration/fleet.go | 10 +-- .../pkg/agent/program/program_test.go | 4 ++ .../pkg/agent/program/supported.go | 2 +- .../endpoint_basic-endpoint-security.yml | 10 ++- .../agent/program/testdata/endpoint_basic.yml | 9 ++- .../testdata/fleet_server-fleet-server.yml | 16 +++++ .../agent/program/testdata/fleet_server.yml | 34 ++++++++++ .../single_config-endpoint-security.yml | 13 ++-- .../testdata/single_config-fleet-server.yml | 3 + .../agent/program/testdata/single_config.yml | 12 ++-- .../elastic-agent/pkg/agent/transpiler/ast.go | 24 ++++++- .../pkg/agent/transpiler/rules.go | 47 +++++++++++--- .../pkg/fleetapi/client/client.go | 18 ++---- .../pkg/fleetapi/client/client_test.go | 10 +-- .../pkg/fleetapi/client/round_trippers.go | 4 +- .../pkg/fleetapi/enroll_cmd_test.go | 6 +- .../elastic-agent/pkg/fleetapi/helper_test.go | 4 +- .../pkg/{kibana => remote}/client.go | 50 +++++++-------- .../pkg/{kibana => remote}/client_test.go | 63 ++++--------------- .../pkg/{kibana => remote}/config.go | 16 ++--- .../pkg/{kibana => remote}/config_test.go | 2 +- .../pkg/{kibana => remote}/round_trippers.go | 22 +------ x-pack/elastic-agent/spec/endpoint.yml | 8 --- x-pack/elastic-agent/spec/fleet-server.yml | 14 ++--- 31 files changed, 288 insertions(+), 249 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server-fleet-server.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server.yml rename x-pack/elastic-agent/pkg/{kibana => remote}/client.go (84%) rename x-pack/elastic-agent/pkg/{kibana => remote}/client_test.go (85%) rename x-pack/elastic-agent/pkg/{kibana => remote}/config.go (85%) rename x-pack/elastic-agent/pkg/{kibana => remote}/config_test.go (98%) rename x-pack/elastic-agent/pkg/{kibana => remote}/round_trippers.go (86%) diff --git a/x-pack/elastic-agent/.gitignore b/x-pack/elastic-agent/.gitignore index cd297650b08d..1051bbdd747a 100644 --- a/x-pack/elastic-agent/.gitignore +++ b/x-pack/elastic-agent/.gitignore @@ -2,6 +2,9 @@ build/ elastic-agent elastic-agent.dev.yml +elastic-agent.yml.* +fleet.yml +fleet.yml.lock pkg/agent/operation/tests/scripts/short--1.0.yml pkg/agent/operation/tests/scripts/configurable-1.0-darwin-x86/configurable pkg/agent/operation/tests/scripts/servicable-1.0-darwin-x86/configurable diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 213454408136..31db04aa7ef4 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -8,6 +8,7 @@ ==== Breaking changes - Docker container is not run as root by default. {pull}21213[21213] +- Read Fleet connection information from `fleet.*` instead of `fleet.kibana.*`. {pull}24713[24713] ==== Bugfixes - Fix rename *ConfigChange to *PolicyChange to align on changes in the UI. {pull}20779[20779] diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 68902b0de726..94a8a826de49 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -83,12 +83,12 @@ func newManaged( return nil, err } - client, err := client.NewAuthWithConfig(log, cfg.Fleet.AccessAPIKey, cfg.Fleet.Kibana) + client, err := client.NewAuthWithConfig(log, cfg.Fleet.AccessAPIKey, cfg.Fleet.Client) if err != nil { return nil, errors.New(err, "fail to create API client", errors.TypeNetwork, - errors.M(errors.MetaKeyURI, cfg.Fleet.Kibana.Host)) + errors.M(errors.MetaKeyURI, cfg.Fleet.Client.Host)) } sysInfo, err := sysinfo.Host() diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go index 71347407768e..968af23c9fef 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go @@ -12,21 +12,20 @@ import ( "sort" "time" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline/actions" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage/store" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client" - - "gopkg.in/yaml.v2" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage/store" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) const ( @@ -106,33 +105,33 @@ func (h *PolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config) if err != nil { return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig) } - if kibanaEqual(h.config.Fleet.Kibana, cfg.Fleet.Kibana) { + if clientEqual(h.config.Fleet.Client, cfg.Fleet.Client) { // already the same hosts return nil } // only set protocol/hosts as that is all Fleet currently sends - prevProtocol := h.config.Fleet.Kibana.Protocol - prevPath := h.config.Fleet.Kibana.Path - prevHosts := h.config.Fleet.Kibana.Hosts - h.config.Fleet.Kibana.Protocol = cfg.Fleet.Kibana.Protocol - h.config.Fleet.Kibana.Path = cfg.Fleet.Kibana.Path - h.config.Fleet.Kibana.Hosts = cfg.Fleet.Kibana.Hosts + prevProtocol := h.config.Fleet.Client.Protocol + prevPath := h.config.Fleet.Client.Path + prevHosts := h.config.Fleet.Client.Hosts + h.config.Fleet.Client.Protocol = cfg.Fleet.Client.Protocol + h.config.Fleet.Client.Path = cfg.Fleet.Client.Path + h.config.Fleet.Client.Hosts = cfg.Fleet.Client.Hosts // rollback on failure defer func() { if err != nil { - h.config.Fleet.Kibana.Protocol = prevProtocol - h.config.Fleet.Kibana.Path = prevPath - h.config.Fleet.Kibana.Hosts = prevHosts + h.config.Fleet.Client.Protocol = prevProtocol + h.config.Fleet.Client.Path = prevPath + h.config.Fleet.Client.Hosts = prevHosts } }() - client, err := client.NewAuthWithConfig(h.log, h.config.Fleet.AccessAPIKey, h.config.Fleet.Kibana) + client, err := client.NewAuthWithConfig(h.log, h.config.Fleet.AccessAPIKey, h.config.Fleet.Client) if err != nil { return errors.New( err, "fail to create API client with updated hosts", - errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts)) } ctx, cancel := context.WithTimeout(ctx, apiStatusTimeout) defer cancel() @@ -140,19 +139,19 @@ func (h *PolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config) if err != nil { return errors.New( err, "fail to communicate with updated API client hosts", - errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Client.Hosts)) } reader, err := fleetToReader(h.agentInfo, h.config) if err != nil { return errors.New( err, "fail to persist updated API client hosts", - errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Client.Hosts)) } err = h.store.Save(reader) if err != nil { return errors.New( err, "fail to persist updated API client hosts", - errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Client.Hosts)) } for _, setter := range h.setters { setter.SetClient(client) @@ -160,7 +159,7 @@ func (h *PolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config) return nil } -func kibanaEqual(k1 *kibana.Config, k2 *kibana.Config) bool { +func clientEqual(k1 remote.Config, k2 remote.Config) bool { if k1.Protocol != k2.Protocol { return false } diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go index 688f8423a4fd..1d77e5022c59 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go @@ -32,37 +32,50 @@ func InjectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag if err != nil { return err } - token, ok := transpiler.Lookup(ast, "fleet.access_api_key") + fleet, ok := transpiler.Lookup(ast, "fleet") if !ok { - return fmt.Errorf("failed to get api key from fleet config") - } - - kbn, ok := transpiler.Lookup(ast, "fleet.kibana") - if !ok { - return fmt.Errorf("failed to get kibana config key from fleet config") + return fmt.Errorf("failed to get fleet from config") } + // copy top-level agent.* into fleet.agent.* (this gets sent to Applications in this structure) agent, ok := transpiler.Lookup(ast, "agent") if !ok { - return fmt.Errorf("failed to get agent key from fleet config") + return fmt.Errorf("failed to get agent key from config") + } + if err := transpiler.Insert(ast, agent, "fleet"); err != nil { + return err } + // ensure that the agent.logging.level is present if _, found := transpiler.Lookup(ast, "agent.logging.level"); !found { transpiler.Insert(ast, transpiler.NewKey("level", transpiler.NewStrVal(logLevel)), "agent.logging") } + // fleet.host to Agent can be the host to connect to Fleet Server, but to Applications it should + // be the fleet.host.id. move fleet.host to fleet.hosts if fleet.hosts doesn't exist + if _, ok := transpiler.Lookup(ast, "fleet.hosts"); !ok { + if host, ok := transpiler.Lookup(ast, "fleet.host"); ok { + if key, ok := host.(*transpiler.Key); ok { + if value, ok := key.Value().(*transpiler.StrVal); ok { + hosts := transpiler.NewList([]transpiler.Node{transpiler.NewStrVal(value.String())}) + if err := transpiler.Insert(ast, hosts, "fleet.hosts"); err != nil { + return err + } + } + } + } + } + + // inject host.* into fleet.host.* (this gets sent to Applications in this structure) host := transpiler.NewKey("host", transpiler.NewDict([]transpiler.Node{ transpiler.NewKey("id", transpiler.NewStrVal(hostInfo.UniqueID)), })) - - nodes := []transpiler.Node{agent, token, kbn, host} - server, ok := transpiler.Lookup(ast, "fleet.server") - if ok { - nodes = append(nodes, server) + if err := transpiler.Insert(ast, host, "fleet"); err != nil { + return err } - fleet := transpiler.NewDict(nodes) - err = transpiler.Insert(rootAst, fleet, "fleet") + // inject fleet.* from local AST to the rootAST so its present when sending to Applications. + err = transpiler.Insert(rootAst, fleet.Value().(transpiler.Node), "fleet") if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go index 2f9f98b59751..fa8d10ad9034 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go @@ -33,8 +33,8 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" fleetclient "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) const ( @@ -60,7 +60,7 @@ type enrollCmd struct { options *enrollCmdOption client fleetclient.Sender configStore saver - kibanaConfig *kibana.Config + remoteConfig remote.Config agentProc *process.Info } @@ -90,13 +90,13 @@ type enrollCmdOption struct { FleetServer enrollCmdFleetServerOption } -func (e *enrollCmdOption) kibanaConfig() (*kibana.Config, error) { - cfg, err := kibana.NewConfigFromURL(e.URL) +func (e *enrollCmdOption) remoteConfig() (remote.Config, error) { + cfg, err := remote.NewConfigFromURL(e.URL) if err != nil { - return nil, err + return remote.Config{}, err } - if cfg.Protocol == kibana.ProtocolHTTP && !e.Insecure { - return nil, fmt.Errorf("connection to Kibana is insecure, strongly recommended to use a secure connection (override with --insecure)") + if cfg.Protocol == remote.ProtocolHTTP && !e.Insecure { + return remote.Config{}, fmt.Errorf("connection to Kibana is insecure, strongly recommended to use a secure connection (override with --insecure)") } // Add any SSL options from the CLI. @@ -162,7 +162,7 @@ func (c *enrollCmd) Execute(ctx context.Context) error { } } - c.kibanaConfig, err = c.options.kibanaConfig() + c.remoteConfig, err = c.options.remoteConfig() if err != nil { return errors.New( err, "Error", @@ -170,7 +170,7 @@ func (c *enrollCmd) Execute(ctx context.Context) error { errors.M(errors.MetaKeyURI, c.options.URL)) } - c.client, err = fleetclient.NewWithConfig(c.log, c.kibanaConfig) + c.client, err = fleetclient.NewWithConfig(c.log, c.remoteConfig) if err != nil { return errors.New( err, "Error", @@ -366,7 +366,7 @@ func (c *enrollCmd) enroll(ctx context.Context) error { errors.TypeNetwork) } - fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.kibanaConfig) + fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.remoteConfig) if err != nil { return err } @@ -680,11 +680,11 @@ func createFleetServerBootstrapConfig(connStr string, policyID string, host stri return cfg, nil } -func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*configuration.FleetAgentConfig, error) { +func createFleetConfigFromEnroll(accessAPIKey string, cli remote.Config) (*configuration.FleetAgentConfig, error) { cfg := configuration.DefaultFleetAgentConfig() cfg.Enabled = true cfg.AccessAPIKey = accessAPIKey - cfg.Kibana = kbn + cfg.Client = cli if err := cfg.Valid(); err != nil { return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd_test.go index 8f5b7c4f8a57..195969ecf58a 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd_test.go @@ -155,9 +155,9 @@ func TestEnroll(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-access-api-key", config.AccessAPIKey) - require.Equal(t, host, config.Kibana.Host) - require.Equal(t, "", config.Kibana.Username) - require.Equal(t, "", config.Kibana.Password) + require.Equal(t, host, config.Client.Host) + require.Equal(t, "", config.Client.Username) + require.Equal(t, "", config.Client.Password) }, )) @@ -215,9 +215,9 @@ func TestEnroll(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-access-api-key", config.AccessAPIKey) - require.Equal(t, host, config.Kibana.Host) - require.Equal(t, "", config.Kibana.Username) - require.Equal(t, "", config.Kibana.Password) + require.Equal(t, host, config.Client.Host) + require.Equal(t, "", config.Client.Username) + require.Equal(t, "", config.Client.Password) }, )) @@ -275,9 +275,9 @@ func TestEnroll(t *testing.T) { require.NoError(t, err) require.Equal(t, "my-access-api-key", config.AccessAPIKey) - require.Equal(t, host, config.Kibana.Host) - require.Equal(t, "", config.Kibana.Username) - require.Equal(t, "", config.Kibana.Password) + require.Equal(t, host, config.Client.Host) + require.Equal(t, "", config.Client.Username) + require.Equal(t, "", config.Client.Password) }, )) diff --git a/x-pack/elastic-agent/pkg/agent/configuration/fleet.go b/x-pack/elastic-agent/pkg/agent/configuration/fleet.go index 60a6b7ced22e..43b7a95e1fab 100644 --- a/x-pack/elastic-agent/pkg/agent/configuration/fleet.go +++ b/x-pack/elastic-agent/pkg/agent/configuration/fleet.go @@ -6,7 +6,7 @@ package configuration import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" fleetreporterConfig "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/fleet/config" ) @@ -15,7 +15,7 @@ import ( type FleetAgentConfig struct { Enabled bool `config:"enabled" yaml:"enabled"` AccessAPIKey string `config:"access_api_key" yaml:"access_api_key"` - Kibana *kibana.Config `config:"kibana" yaml:"kibana"` + Client remote.Config `config:",inline" yaml:",inline"` Reporting *fleetreporterConfig.Config `config:"reporting" yaml:"reporting"` Info *AgentInfo `config:"agent" yaml:"agent"` Server *FleetServerConfig `config:"server" yaml:"server,omitempty"` @@ -33,8 +33,8 @@ func (e *FleetAgentConfig) Valid() error { return errors.New("empty access token", errors.TypeConfig) } - if e.Kibana == nil || len(e.Kibana.Host) == 0 { - return errors.New("missing Kibana host configuration", errors.TypeConfig) + if len(e.Client.Host) == 0 { + return errors.New("missing fleet host configuration", errors.TypeConfig) } } @@ -45,7 +45,7 @@ func (e *FleetAgentConfig) Valid() error { func DefaultFleetAgentConfig() *FleetAgentConfig { return &FleetAgentConfig{ Enabled: false, - Kibana: kibana.DefaultClientConfig(), + Client: remote.DefaultClientConfig(), Reporting: fleetreporterConfig.DefaultConfig(), Info: &AgentInfo{}, } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 19eb54991977..2691aae9e418 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -395,6 +395,10 @@ func TestConfiguration(t *testing.T) { // programs: []string{"journalbeat"}, // expected: 1, // }, + "fleet_server": { + programs: []string{"fleet-server"}, + expected: 1, + }, "synthetics_config": { programs: []string{"heartbeat"}, expected: 1, diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 26e4368897e0..a893c7858cb3 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -25,7 +25,7 @@ func init() { // spec/metricbeat.yml // spec/osquerybeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzkWll3o7qWfu+fcV5vDwxx7qXXug82OUx2SBlXJKE3JNlgW2BOjLGhV//3XmIyYCdVqTpD9+qHrCRYaNjaw/d92//1yzFd0/8I0vjfjuu3fP3270XMf/nPX0hsZPjrIVyCmbsALqcJ5jRMdwQuH23TOJOVXGLkKBjZcx85UgBx5Kt3P0toeQhhYWfeyj7aupP5cBJhBWQYTqRFDE4+dI4YLjVmOTKuxrw3Vs6x+aqtdfnsQ/dtAfERQyDZ23Nob2Wj+h0P1j9h05B8oJXMcrgP5fJ2PYfpiSMTE5Qv4SG0dSn0Fe28BppEZO0YIFeqn09DW5+lzATZy3YWExNwNu2eS6Q8hAGcnBnySn07rZ+b2gkpbk5ifAygK71sZyeiaGfx+WI1y3w0fezGWrOImeGjbdZnuj7v7615pkshjUFGVMyRkvH118P8+ln9Eyhg8rKdRb7icqq6Gx/N0mrs8ofmKTCa5TTxUhLT/pjMthxOoKZgoL1htL+ep/0xq3lDHzJOkuW8esfE6dpo70R6tK1Ma2wSB/AiYeRsWGwcGeyfe1ZieOG+6uV0d8/W9TrM4mfcnXGm+PAiY/Q82NdiNYuoKXV7IZbH6e56dqqAI4auRFTnxu4369bz5Qx5Z4aWQ9u0d5mwA4YPj7Z54SRmUqCH+7XCT9QCElWl1H56CJ/1WUTiZRiYRrlSwGSue38nKpDEmM3qHDoKOPrIlQLolhgaha+EyXx5+Ocv/1oH8Dph6WGbZKPw9eBkT00tJckyfFXAjiEnZdZ+7ivy/mU74yT2zkThJ6bLJYauTGMurZdpJK4ax8aOPR1CfJ0jwyZQ9KRKB6mvvD7aT7768hTOiaklSBUuHNUmM72IJiwlu0Nob7XnADqFj5zJQmqP8Zz39pZT1YuY+ZqLeRYKOGFrlgci5FeHk3hmV3NeUpKAh5ftdLtQtDPTNYOYRslMvltIvXdUV/KRxxfKJceF1juj9NsiFs/sua3P1ABO9kRlpZhvWaYUGbOCKKzwoRR6MT9i5FL0a2d28Xe3BjIuJTMNCYMLrc5uXO6u4ysR95VsE8CJGH8kT4f5YjXjaxPskIJTYr42rjk7+8g7iL307U2vd7ZtxkU0Zp1bLlbTLYtBEUA8sXvPFiv5SBTavDPNbN0pmelxmtjtPFIAZS5c7GU7LZ+n6ZmoroSEW6peRMzzo76VQowi7suaCEverklNQwqeDqEd92yOXO6roAiQ1+2jSf3zNnTsuJu7ty87W8D2TppxiSthExR023u2lTKGZgmNjT1eDZ/TGJREBYWvgLJvg3fsOBi/SNKU6s18lpcSCHKGlsKvz8ImNAYbBicpSVzJh5fjS3jIbBM8YOhusPCRNmW26Vt37sdUu2fTKLDaT0fivODU+Ue7l/v3eN8+t/suGLxwYfsqpaJoQ1WvwNAQpevvm2Xz/J1Ue01/9qNtNmnqfOiXA2mNZrzxo3GKFfbp/EKv7NWkWK6Nxn6U0mdVORFzNeW6OSuOiAV436+qMl6fqZ1PxJsoyc07Licm2DFTKyoootRpeREbO+EDVNYiavKNONtCb0q7PnkLoBOR2GDD+Ow+j6i5n4sYIdA4j2MDxxeO23IwKuGL1azAUM5ZDKo1ByWmK5fuDiO3RIpxDurSOv92yW5tlqV+bJz85fevRxVQiDwyuK/dXft1+xic8bY0ZrapycyayS1MqvaBcEoVnpPwMGdKxEV9IMJfVO8gSl49p6fNV9O/2U/T0IeT/e9fNtMdUSYCfkTCT4RPOFZWMDipcuIiFuMizdZ/1WydRTSWFFen87bUbrZ8TdbBTakV6QA63EfLtrxWadOPQcSmaZ22tzPSIdXE5cwC50XMj2Q14SQ2tsQE+y9QuLjLB6i2HZt4nKDZsSpvPSSLY+NIldftQp9uF6/1bwKNU4W6IDgxfZIRxeNfUJhR09gFhdy48Udo+0NkfiQKSwI4SRbxhbMYHL9Aj/sJSGwujRiBsIlXLqpyB7YYGlIPcb+DBit3+k2kIVGSsAkeWretEN5X4TZaQs9V2KckTquwb9McUkUpr5Bv3rnxLRoXaZ3TxNv4EEuCrTTpdELNV4HWcqw+vxvio3TVpdw2HBoWERHzsmGmtiEmL9nTIFwr1tDuuU2x32IO7RiMoh1GM6nyqcSVaAwigp6ruw/gsvqN4STyRUhV9+ycaaxVaUUgW3FPo73eYTxiL1zC1b30bJo8/+g5rjaPQUxUh1cpR7CIOo6au8IFUaSb8sPMf1yfqd2Z5w1UkaiArEZ9BqSIfcvle/c23m+APE6+3p5jsGbtbz3m1pWFlCQzmVnP76V8sbcTVS4C5g7YSLuvIXMRNpAjas0GzKR+fslxAx2rv/v2rvwCC6aVN/CkipMhK5uJeD0xXROQsILoVPX2AXwYrQOUKg+o3o6K/Znu+Z15ZGxNH20L7Ol0uJcaznu5r2TiHCE2tV2ggGI0j4A4OY3BPkDuhiqXnAnoLnyqevZ8e/5CK9fIFe892pY7Ee+0dviecseQy5Fypzx9472hmiB9oiQLemN0+WkRTyICQSlyMf5EmR6tX6sbyOVs+kdDOkOlhbwjprhrFjHTPYw+K5+v9o/WCSjwSq79w4xk/xoH9RwmzpmA58m+n4NOzDRSEnf+kdlm4zvX9xNfnWbUAluqgiuNMCOJWbNND/JfYYnlSdRMS6I8XJ8pRhwov3b/92Iks035wqzr+yQGEo4vObueL38ufdlHwjeXfR/o+ao0iinxPy8H61jejl59T8TQFdLCSbq+fiaoqvD1HvSq6Gmd/34eznfY4937t7yCwdf5sC4bicgPVX2O/1HX7aqeuzm1WiVoFtNYy27zo5cP8VH/DPYtPUj6vtjDEe9DzTeMeCkgK16mamCC08t2dsRwkjAzPDhWdl1bnzR7c0ewM4qolPH1SsDOZn5LOto6463sQBUQ0dg9OMU5dFSHY5OXTnEWZ0waH0sW+jSpsIrqRXSk2mz4ep3dF169Gl2Hry0yj90MX5lF1iH2uGaUtnEUbFHAN8E8K7h3C8Ua5oM6QbcPW+5DsUYUFS7HTFAgdczAPnaxLgXWsKp751bgG7CKrMdIfp/1x0zygz20jPJDgbNJyS3baPfZ7kWEhYAOVyF3xEKLjkl2YnU71yK+YS0huoqaLeToRMdGPL3PfseCpoCLrR8kjR9sJ2eiXFJf3Z8CuLy3Vhu+p2e9G/u5de/f86m950Uc5VS9L6AO9qk60kh8vbEjVd3jrbDe+EXxcO7mC1u/lyPag+Z3Wbx4noDjHTH5/bL9Df/7nID+niLwuXkChW8FzfuhM1gOp6ZWMJOfhrDzuxsJXWPo4ybITQyP6NXnznwP7n1y33sfeRFSu5L783OsJicfypyqs8hXXn/oXB/AwR+6m7ZJNs65d5swyU0cjJpi0+9qmNxvjtRxv1ntwy/b6dk2jRPWB+X7dyjZ+07RidbBW3ZH0lmZIKKJV0ONpv4Gg2e92tvILlQBEkPTUwAv2bckmnYsM0FGzYoanjrK/iTHPryUP98QlSMSC6gmC1rYn1+iCRjKTLGgliwlMT2Riv6dNWyCLYN0i0ZdoQo6Wc/5wB6jRiqO+UPn718P4VodNd06RdjdMIVLgaEVGDK+tqY3dXbU2CvE/aDEnZDEO2Dkpb76nC+2x+/ABH8JlviReBpCZv0+1L1SGjChYp749a6y2sbhKKYOPnIXGO1FTDUx6WkCsmJ4iagq7OpyHzm7QKcijgoMvZQWtKqvjpJFOM6iCv4KCl513oaxFa+zty29E1xfIZBozHeNPtp8s0DmzHJSX2l01NtvD5QYeTIVmN2UvhUs7VgJQ/lMTEPC39JZR8FCoLbHX+WHBRJJ+pg1gOAjnfU6P6o403CsqSVY8KVicqx0mid5j6Ej48JhIpkwk8d+rZNVAUULLcPIKwLoNgE2y2kNtLp2YgNgquI9aEkNWnVyjq0KvJywXmknEobSaQ3lo311RknYG6Plo0g0RPGqYF7Ey1xwV5GkFgnPiD7ZB8htdcX5FRTdD/i+PhvAyR6jsOV3lebwsp21Zyxr/shPQQx2rCMi8oZaTl61ahStCx6iTDa+op1wLMDosg5WASYNLcKJ12kGnV7c+Fuj4RXCdwh8/R6e2habfoEcgsYxcOlpk/XnN3rgjqizCVKMIzHe0V3rta9r9pLD7dknOVGnfd2Jr02XU2tZFadO0yyquEgbPbjz1brHMNB3t2g52qvq5Ui5pFRdDhJVp5v27migAX/qHN0dbjHEVTL7k7XdGx0FqSxlZrShMUgwiro+wB3drC5K24e3hdLkMfV5/2Gr889tj/6kbg3eL9If6diWI2J8PX/Sll9qEeBvi+0xvbVRU0jFGk+H0BkIBhWIrQDzUBdt+gQjcaHtSYj8LYhDT9uLAgVsfOQU/ghYtz7S5QkFDIhm7Svtnl1+Lczfo//23vuM3tzWt79Eo67+L/FQ4/xTdO4RYRvUkPqbZFUticUdixojckTVw/qu/tEQY4h3x76FEyfv+8JQJxzUhq5OfqZ1Ppj3erejb5ENQd19ojQ4y6n1959pjy+q/VyBnq2zNx/iN39V/X0kChPYowx0murhPzst83D87bR+K+4hPdW9MAiK9bA7nlPVkDFyJv8vO+RQWF07oeYLDUgVGaztiGslRYDTZD//v9Ypf5da9SLhpurcoXhjOiWio/WXz0TaB19K+aNoUqrHbk6qjryRksQrX7bT/Vp1ZJJ4KYGvp2qcJYX2Vyl0FKMgX33JOV8jKQ3ofn1Pj3gVjqkAaUCZLEFNMs7MMWWimVeH7DcokxhzM/ZDylTxvUI2at73XZSpaoMsXl/rdsjHlGk49l3KxN6jTJUWgtG7msSf0segzV21Wto32mZ/hD7wv0IHqErEf//L/wQAAP//gopK0A==") + unpacked := packer.MustUnpack("eJzkWll3sziavp+fUbc9C0ucbuacvjBOs4WQz/iLJHSHJBtsC0zFGBvmzH+fIzYDdvIt1VU1c+YiJ4kQ0ivpXZ7nEf/1yzFb0/8Is+Tfjuv3Yv3+72XCf/nPX0hi5PjrIVoC3XOBx2mKOY2yHYHLR9s0zmQlVxg5Ckb2c4AcKYQ4DtS7z1JaHSJY2rm/so/2wskDOIuxAnIMZ5KbgFMAnSOGS41ZjozrPh/1lQtsvmnrhXwOoPfuQnzEEEj29hzZW9mofyej+U/YNKQAaBWzHB5Aubqdz2GL1JGJCarX6BDZCykKFO28BppEZO0YIk9q2ueRvdAzZoL8dasnxASczft2iVSHKISzM0N+tdjOm3ZTOyHFK0iCjyH0pNetfiKKdhbP3ZWeB2j+2Pe19JiZ0aNtNmu6tg9ta9sWUkQTkBMVc6TkfP318Hx91vyECpi9bvU4UDxOVW8TID2r+y5/apwSI72gqZ+RhA775LblcAI1BQPtHaP9dT3dj1mPGwWQcZIun+t3TJytje5MpEfbyrV2T5IQXiSMnA1LjCODw3XrFYYXHqh+QXf39rqZh1n8jPs16koALzJGLyO73JUeU1PqbSGWz+nuunaqgCOGnkRU52bfb+ZtxisY8s8MLcd7051lyg4YPjza5oWThEnhItqvFX6iFpCoKmX200P0stBjkiyj0DSqlQJmzwv/r0QFkuizWZ0jRwHHAHlSCL0KQ6MMlCh9Xh7+/su/NgG8Tll22Kb5JHx9ONtTU8tIuozeFLBjyMmYtX8OFHn/utU5SfwzUfiJLeQKQ0+mCZfWyywWR40TY8eeDhG+jpFjEyiLtE4HWaC8PdpPgfr6FD0TU0uRKlw4brbM9GOasozsDpG91V5C6JQBcmau1C3jpRjYVlDVj5n5VohxXAWcsKUXoQj51eEk2ux6zEtGUvDwup1vXUU7s4VmENOomMl3rjR4R/WkAPncVS4FLrXBGqVf3US02c/2QldDONsTlVVivGWVUWToJVFYGUAp8hN+xMij6B/9tou/+zmQcamYaUgYXGi9duNyd55AiXmg5JsQzkT/I3k6PLsrna9NsEMKzoj51rqmfg6QfxC2DPebXs9s2/aLacJ6t3RX8y1LQBlCPLMHbe5KPhKFtu/Mc3vhVMz0OU3tbhwphDIXLva6nVcv8+xMVE9Cwi1VPybm+XGxlSKMYh7ImghL3s1JTUMKnw6RnQz2HHk8UEEZIr+3o039z13o2Ek/9sAuO3dhdyZtv9STsAlKuh20baWcIT2libHHq3E7TUBFVFAGCqiGe/DBPo76u2mW0UU7nuVnBIKCoaXw67PYE5qADYOzjKSeFMDL8TU65LYJHjD0Nlj4SJcyu/S9cO7HVGezaZRYHaYjsV5w6v2js+X+Od7fn1u7SwYvXOx9nVJRvKGqX2JoiNL1182ybf8g1V7Tn/1om22aOh+G5UBaI523fjRNsWJ/er9Y1PvVpliuTfp+ltL1upyIsdpy3a4Vx8QCfOhXdRlv1jQoqV08jUs1TcFxZOsn5atdaw9l6nhI+JGZoLzdK2cQk+M53ZWekVSXmfXS29yl9qEdLOFVCLXTIsrU0ASn161+xHCWMjM6OFbejGmNywURe636B1Euuv3brPbRl+38bJvGCS/0Q4A8F6O9GKMtU77mLuYphpeYqn4WqB4PkLMLFzRbJF4hfJ0mRkZSX+TG/Vp1ZCJKP3w71f0sKbK/SpGjGCX5GkjO+VqGNlu+JuvwpgyJUIEOD9CyKz11SgkSELN51qS0rU56FJd6nFng7Cb8SFYzThJjS0yw/wLF8Xt8hPi6vqnPCdKPdeofoDycGEeqvG3dxXzrvjW/CTRONSKB4MQWs5woPv+Copyaxi4s5WarF58h0U9R65EoLA3hLHWTC2cJOH6BPg9SkNpcmqBlsSd+5dalAGwxNKQBGv0AKdUo5FfhdiJdYxM8dK5Xo5+vh0iUJHquQyIjSSZCrE8BSBVlrkaFRY96bpGqSHmcpv4mgFgS7t+mmhk13wSSKbD6UocEgcZ5Whomodyno879W4QdE/OyYaa2ISav2NMQfTaIurO5Sz/fQtVdH4ziHUa6VPtU7c4gJuilPvsQLuvfGM7iILlwXJ+zc6aJtsPIq0Qoi3Oa2HqHDQhbuITrcxnsafrys+u47nkCEqI6vE6fIkU1cdSeFS6JIt2kZmb+7dqm9mt+bsu4COkdM5o1IEXYLVcfndvU3hD5nHy9XcdozvOHKXic/iaMxl3pwrYTVS4CAo5ScGfXGNWLPZBjaukj1N60Xwrcwqr67+F+136BBQsp2tJdx8mYsegiXk9soQm4VMNXqvr7ED5M5gFKnQdUf0eFfaZ3/mAcGVvzR9sCezof29JAXb8IlFysI8KmtgsVUE7GEeW/oAnYh8jbUOVSMAFrhU/VbS+36y+1ao088d6jbXkz8U63D+5KLzGUC5aATV3eRr7cMijkcaRoMrN0eVDCvvnemGlf4+djhtmVeAH9jT4/ucksJhBUIhd3Z/4T8zfMH3miDP/OcMdQaSnviCnOmsXM9A6TZ9XLdf/jdQpKvJIb/zBjObjGQTOGiQsmoGu6H+agEzONjCS9f+S22frO9f00UOc5tcCWquAKsc1YYpa+GcDhftzQ8iVqZhVRHq5tipGEyj+uUOUaI7ltyhdmXd8nCZBwcinYdX3FSxXIARK+uRz6wMBXpUlMif95NZrH8nf06nsihq5wD86y9fWZoHHC1/vnuKFuTf777VC3xx4fnr/llwy+PY/rspGK/FDX5+RvTd2u67lXUKuDmXpCEy2/zY9+McZHwzXYt9A5HfriAEf0bTex/I4Rr5iplXh5H2r2cy9mrW2e9rya/8V+mkcBnO1tM46plPP1Ktqvu1xhSUd7wXhHyakCYpp4B6c8R47qcGzyyinPYo1p62OpgKE1VlH9mE4UjQ1fr/P7oqQvyjaqFY1SsN9F4uX46RDhpl2wvba9YVu2cRRMSsA3wcpquHcLxRqxRjDZVuwcwpb7UKwVDDtWgNQpO/ncxfoU2MCq/p1b8atz7cn6/lnzT1nWJzZ0bOtT9tSm5J7ltHZ2toiwENDhKnJ6nJhgJ1yyEYtbBrWY9UJuN5ab3AhhEboKfh3k6AW5VljswmUjWCm5uz9OQ0E6P0hbP9jOzkS5ZIG6P4VweW+uLnxPL4u+74/Ne/+cT905u0lcUPW+uDiyU3WkiTB5s49U9Y63onPrF+XDuR8v6vxejukAmneQrLXx2j5m1v3847Q1+Bmf+Y3w3JbPOo3evNuMfSCq1/vWkIZ8CD0/tvNjeDGKU1BhoN0Xe79TCL8Pk37TGKc6LyLvHECP/9S6biBY/X+Ff3KNA5rwUyL71LeaMaa5az5WUv5M1ST6e1+24nX4nt+RQFYmiGnqN6W5rVfhqG1Qq1qZgipAYmh+CuEl/5ak0fVlJsipWVOpU09xn+QkgJfqt1+uyTFJREzKgkYNx5doCsayTCKoGMtIQk+kpktnDZtgyyDdoskNQw01rJditB+TSzmc8AfU0dqvh2itTi5wenXR2zCFS6GhlRgyvrbmN3VpcklUivNBqTcjqX/ASJztS+Fuj99RQ/+U2jua9+4FWXqT327y4D1oeKUAYEbFOMnbM1NiLmjwPzlmBFQsMfQzWtK6HjlKHuMkj2u4KChrfYsj4OP+uYutZJ2/b+md4PoKgUQTvmv1xPaWWubMcrJAaXXH25voCiNfpgLjmtK3gqXrK2Eon4lpSPhbuuQkWAjU9vir/OAiPQ6UY94muc90yev4qOYY476mlmLBL8rZsdY1nuQ9ho6MS4eJZMJMngSNrlQHFC21HCO/DKHXBphe0AaY9FdTbcGvC8boemN07SMX2KqL6Akvaq1BwlA6raF8tK/OKIn9xmj5KBINUfw6mN1kWQiuJ5KUm/KcLGb7EHmdDvd8Lc73A36oZ4Zwtsco6vhQzdFft3q3xqrhW/wUJmDHeuAub6jlFIECKqpoffAQZbYRHBsnArwtm2AV4MvQYpz6Pcfu9dXW31rNqxS+Q+Db9/C6jk/1XyQ0dg5A1hSUDbQ8+75+tiOqPkOKcSTGBzplM/d1zkFyuF37rCDqfKjT8LXpcWot6+LUF/eyjous1U97X200+ZEeukXLia2qXyDlklF1Ob5y6XTGwRmNCv4PraM/wy2GuE5mf7AWeqM7IJVlzIw3NAEpRnEPWO/oTE1R2j68u0qbx9SX/afXZn/+VdsP6Lzg4yL9me5rOSLG189P2vJLQ5r/4m6P2e0etYVUzPF0iJwRwa71qFMAZT7WEVtdfULGOw1f5G8GL3yghcWhAjYBcspgenXY+kifJxQwIjGNr3Q2e3x4NfhtvXTw3o/os119+1M03Smh+ON04YmePaohzVdJdS1JxBmLGiNyRH3n8133LWOMId6d+hZOnWLoC2NdbVQb+jo5iuWONN0lzNJ43OvZTkjqGNTd/yJptJZT5++/5Qslt7bnCvTsBXsPIH4PVvXfR6IwgT2qKYk6HH89rd/Le0hP9S4MgnI9vk0uqGrIGDmz/5c3ylDsunZC7TdlSBUZrLtB1iqKAKfp/vn/2s3yh9RqEAk3VecOxZvSKREdnb/8SKRNvv0b0qffiyb95AcZ10jKQrpf39Mj3oRjKkAaUSZLUJOcM3NKmWjuNyH7Dcok+tz0/ZQy1XyvlI2G930XZaqvDdy3t+b64HPKNO77IWViH1GmWgvB6ENN4g/R/Wl7Vp3W+Y1rpt9DH/hfoQPUJeK//+V/AgAA///YeLmN") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml index d81d276f3685..dfbec8016ba0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic-endpoint-security.yml @@ -1,16 +1,14 @@ revision: 5 fleet: + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s agent: id: fleet-agent-id logging.level: error host: id: host-agent-id - api: - access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw - kibana: - protocol: https - host: localhost:5601 - timeout: 30s output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml index 1681926c56e8..9f438cd46fdc 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml @@ -1,16 +1,15 @@ revision: 5 name: Endpoint Host fleet: + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s agent: id: fleet-agent-id logging.level: error host: id: host-agent-id - access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw - kibana: - protocol: https - host: localhost:5601 - timeout: 30s outputs: default: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server-fleet-server.yml new file mode 100644 index 000000000000..c03696aff1f0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server-fleet-server.yml @@ -0,0 +1,16 @@ +fleet: + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server.yml new file mode 100644 index 000000000000..8268f258fb58 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/fleet_server.yml @@ -0,0 +1,34 @@ +name: Fleet Server Only +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +outputs: + default: + type: elasticsearch + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: + - id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml index 42d78b09ca53..8e4092d6d63f 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-endpoint-security.yml @@ -1,13 +1,14 @@ fleet: enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s agent: id: fleet-agent-id - api: - access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw - kibana: - protocol: https - host: localhost:5601 - timeout: 30s + logging.level: error + host: + id: host-agent-id output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml index cb8a3e106599..c03696aff1f0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml @@ -1,6 +1,9 @@ fleet: agent: id: fleet-agent-id + logging.level: error + host: + id: host-agent-id output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml index 006db1e9f524..afb0a5b54272 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml @@ -1,13 +1,15 @@ name: Production Website DB Servers fleet: enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s agent: id: fleet-agent-id - access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw - kibana: - protocol: https - host: localhost:5601 - timeout: 30s + logging.level: error + host: + id: host-agent-id server: output: elasticsearch: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go index 13242297c9a6..efadf408c717 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go @@ -1067,11 +1067,33 @@ func Insert(a *AST, node Node, to Selector) error { return fmt.Errorf("expecting Key and received %T", current) } - switch node.(type) { + switch nt := node.(type) { case *Dict: d.value = node case *List: d.value = node + case *Key: + // adding key to existing dictionary + // should overwrite the current key if it exists + dValue, ok := d.value.(*Dict) + if !ok { + // not a dictionary (replace it all) + d.value = &Dict{[]Node{node}, nil} + } else { + // remove the duplicate key (if it exists) + for i, key := range dValue.value { + if k, ok := key.(*Key); ok { + if k.name == nt.name { + dValue.value[i] = dValue.value[len(dValue.value)-1] + dValue.value = dValue.value[:len(dValue.value)-1] + break + } + } + } + // add the new key + dValue.value = append(dValue.value, nt) + dValue.sort() + } default: d.value = &Dict{[]Node{node}, nil} } diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 4fec7c78032f..d9f4fd9e34d8 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -999,15 +999,35 @@ func (r *MapRule) Apply(agentInfo AgentInfo, ast *AST) error { switch t := n.Value().(type) { case *List: - return mapList(agentInfo, r, t) + l, err := mapList(agentInfo, r, t) + if err != nil { + return err + } + n.value = l + return nil case *Dict: - return mapDict(agentInfo, r, t) + d, err := mapDict(agentInfo, r, t) + if err != nil { + return err + } + n.value = d + return nil case *Key: switch t := n.Value().(type) { case *List: - return mapList(agentInfo, r, t) + l, err := mapList(agentInfo, r, t) + if err != nil { + return err + } + n.value = l + return nil case *Dict: - return mapDict(agentInfo, r, t) + d, err := mapDict(agentInfo, r, t) + if err != nil { + return err + } + n.value = d + return nil default: return fmt.Errorf( "cannot iterate over node, invalid type expected 'List' or 'Dict' received '%T'", @@ -1022,7 +1042,7 @@ func (r *MapRule) Apply(agentInfo AgentInfo, ast *AST) error { ) } -func mapList(agentInfo AgentInfo, r *MapRule, l *List) error { +func mapList(agentInfo AgentInfo, r *MapRule, l *List) (*List, error) { values := l.Value().([]Node) for idx, item := range values { @@ -1030,24 +1050,31 @@ func mapList(agentInfo AgentInfo, r *MapRule, l *List) error { for _, rule := range r.Rules { err := rule.Apply(agentInfo, newAST) if err != nil { - return err + return nil, err } values[idx] = newAST.root } } - return nil + return l, nil } -func mapDict(agentInfo AgentInfo, r *MapRule, l *Dict) error { +func mapDict(agentInfo AgentInfo, r *MapRule, l *Dict) (*Dict, error) { newAST := &AST{root: l} for _, rule := range r.Rules { err := rule.Apply(agentInfo, newAST) if err != nil { - return err + return nil, err } } - return nil + n, ok := newAST.root.(*Dict) + if !ok { + return nil, fmt.Errorf( + "after applying rules from map, root is no longer a 'Dict' it is an invalid type of '%T'", + newAST.root, + ) + } + return n, nil } // MarshalYAML marshal a MapRule into a YAML document. diff --git a/x-pack/elastic-agent/pkg/fleetapi/client/client.go b/x-pack/elastic-agent/pkg/fleetapi/client/client.go index 8f9fcd7bc982..86efb3d2d664 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client/client.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client/client.go @@ -15,10 +15,9 @@ import ( "os" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) // Sender is an sender interface describing client behavior. @@ -55,7 +54,7 @@ func init() { return nil, errors.New(err, "could not create the logger for debugging HTTP request") } - return kibana.NewDebugRoundTripper(rt, l), nil + return remote.NewDebugRoundTripper(rt, l), nil } } } @@ -65,8 +64,8 @@ func init() { // - Send the API Key on every HTTP request. // - Ensure a minimun version of Kibana is required. // - Send the Fleet User Agent on every HTTP request. -func NewAuthWithConfig(log *logger.Logger, apiKey string, cfg *kibana.Config) (*kibana.Client, error) { - return kibana.NewWithConfig(log, cfg, func(rt http.RoundTripper) (http.RoundTripper, error) { +func NewAuthWithConfig(log *logger.Logger, apiKey string, cfg remote.Config) (*remote.Client, error) { + return remote.NewWithConfig(log, cfg, func(rt http.RoundTripper) (http.RoundTripper, error) { rt, err := baseRoundTrippers(rt) if err != nil { return nil, err @@ -81,14 +80,9 @@ func NewAuthWithConfig(log *logger.Logger, apiKey string, cfg *kibana.Config) (* }) } -// NewWithRawConfig create a non authenticated clients. -func NewWithRawConfig(log *logger.Logger, config *config.Config) (*kibana.Client, error) { - return kibana.NewWithRawConfig(log, config, baseRoundTrippers) -} - // NewWithConfig takes a Kibana configuration and create a kibana.client with the appropriate tripper. -func NewWithConfig(log *logger.Logger, cfg *kibana.Config) (*kibana.Client, error) { - return kibana.NewWithConfig(log, cfg, baseRoundTrippers) +func NewWithConfig(log *logger.Logger, cfg remote.Config) (*remote.Client, error) { + return remote.NewWithConfig(log, cfg, baseRoundTrippers) } // ExtractError extracts error from a fleet response diff --git a/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go b/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go index 098a314af2ae..db2242ce7a43 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) func TestHTTPClient(t *testing.T) { @@ -37,7 +37,7 @@ func TestHTTPClient(t *testing.T) { "host": host, }) - client, err := kibana.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { + client, err := remote.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { return NewFleetAuthRoundTripper(wrapped, "abc123") }) @@ -66,7 +66,7 @@ func TestHTTPClient(t *testing.T) { "host": host, }) - client, err := kibana.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { + client, err := remote.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { return NewFleetAuthRoundTripper(wrapped, "abc123") }) @@ -91,7 +91,7 @@ func TestHTTPClient(t *testing.T) { "host": host, }) - client, err := kibana.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { + client, err := remote.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { return NewFleetUserAgentRoundTripper(wrapped, "8.0.0"), nil }) @@ -113,7 +113,7 @@ func TestHTTPClient(t *testing.T) { timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() - client, err := kibana.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { + client, err := remote.NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { return NewFleetAuthRoundTripper(wrapped, "abc123") }) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/fleetapi/client/round_trippers.go b/x-pack/elastic-agent/pkg/fleetapi/client/round_trippers.go index a0c4c04e0688..4849edf3276e 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client/round_trippers.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client/round_trippers.go @@ -8,7 +8,7 @@ import ( "errors" "net/http" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) // ErrInvalidAPIKey is returned when authentication fail to fleet. @@ -29,7 +29,7 @@ func (r *FleetUserAgentRoundTripper) RoundTrip(req *http.Request) (*http.Respons func NewFleetUserAgentRoundTripper(wrapped http.RoundTripper, version string) http.RoundTripper { const name = "Elastic Agent" return &FleetUserAgentRoundTripper{ - rt: kibana.NewUserAgentRoundTripper(wrapped, name+" v"+version), + rt: remote.NewUserAgentRoundTripper(wrapped, name+" v"+version), } } diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go index 29bbf7e607e5..54f983066933 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go @@ -16,7 +16,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) func TestEnroll(t *testing.T) { @@ -67,7 +67,7 @@ func TestEnroll(t *testing.T) { "host": host, }) - client, err := kibana.NewWithRawConfig(nil, cfg, nil) + client, err := remote.NewWithRawConfig(nil, cfg, nil) require.NoError(t, err) req := &EnrollRequest{ @@ -103,7 +103,7 @@ func TestEnroll(t *testing.T) { "host": host, }) - client, err := kibana.NewWithRawConfig(nil, cfg, nil) + client, err := remote.NewWithRawConfig(nil, cfg, nil) require.NoError(t, err) req := &EnrollRequest{ diff --git a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go index 4650eb5c4ffc..7aac16dbf976 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go @@ -14,7 +14,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/remote" ) func authHandler(handler http.HandlerFunc, apiKey string) http.HandlerFunc { @@ -47,7 +47,7 @@ func withServerWithAuthClient( return withServer(m, func(t *testing.T, host string) { log, _ := logger.New("", false) - cfg := &kibana.Config{ + cfg := remote.Config{ Host: host, } diff --git a/x-pack/elastic-agent/pkg/kibana/client.go b/x-pack/elastic-agent/pkg/remote/client.go similarity index 84% rename from x-pack/elastic-agent/pkg/kibana/client.go rename to x-pack/elastic-agent/pkg/remote/client.go index 24be2e7f05ea..2c7b36063e64 100644 --- a/x-pack/elastic-agent/pkg/kibana/client.go +++ b/x-pack/elastic-agent/pkg/remote/client.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package remote import ( "context" @@ -23,10 +23,9 @@ import ( ) const ( - kibanaPort = 5601 - kibanaHTTPSPort = 443 + defaultPort = 8220 - kibanaRetryOnBadConnTimeout = 5 * time.Minute + retryOnBadConnTimeout = 5 * time.Minute ) type requestFunc func(string, string, url.Values, io.Reader) (*http.Request, error) @@ -40,7 +39,7 @@ type requestClient struct { lastErrOcc time.Time } -// Client wraps an http.Client and takes care of making the raw calls to kibana, the client should +// Client wraps an http.Client and takes care of making the raw calls, the client should // stay simple and specificals should be implemented in external action instead of adding new methods // to the client. For authenticated calls or sending fields on every request, create customer RoundTripper // implementations that will take care of the boiler plates. @@ -48,14 +47,14 @@ type Client struct { log *logger.Logger lock sync.Mutex clients []*requestClient - config *Config + config Config } -// NewConfigFromURL returns a Kibana Config based on a received host. -func NewConfigFromURL(kURL string) (*Config, error) { +// NewConfigFromURL returns a Config based on a received host. +func NewConfigFromURL(kURL string) (Config, error) { u, err := url.Parse(kURL) if err != nil { - return nil, errors.Wrap(err, "could not parse Kibana url") + return Config{}, errors.Wrap(err, "could not parse url") } var username, password string @@ -75,27 +74,27 @@ func NewConfigFromURL(kURL string) (*Config, error) { return c, nil } -// NewWithRawConfig returns a new Kibana client with a specified configuration. +// NewWithRawConfig returns a new client with a specified configuration. func NewWithRawConfig(log *logger.Logger, config *config.Config, wrapper wrapperFunc) (*Client, error) { l := log if l == nil { - log, err := logger.New("kibana_client", false) + log, err := logger.New("client", false) if err != nil { return nil, err } l = log } - cfg := &Config{} - if err := config.Unpack(cfg); err != nil { + cfg := Config{} + if err := config.Unpack(&cfg); err != nil { return nil, errors.Wrap(err, "invalidate configuration") } return NewWithConfig(l, cfg, wrapper) } -// NewWithConfig takes a Kibana Config and return a client. -func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Client, error) { +// NewWithConfig takes a Config and return a client. +func NewWithConfig(log *logger.Logger, cfg Config, wrapper wrapperFunc) (*Client, error) { // Normalize the URL with the path any spaces configured. var p string if len(cfg.SpaceID) > 0 { @@ -108,10 +107,7 @@ func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Clien p = p + "/" } - usedDefaultPort := kibanaPort - if cfg.Protocol == "https" { - usedDefaultPort = kibanaHTTPSPort - } + usedDefaultPort := defaultPort hosts := cfg.GetHosts() clients := make([]*requestClient, len(hosts)) @@ -139,12 +135,12 @@ func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Clien Timeout: cfg.Timeout, } - kibanaURL, err := common.MakeURL(string(cfg.Protocol), p, host, usedDefaultPort) + url, err := common.MakeURL(string(cfg.Protocol), p, host, usedDefaultPort) if err != nil { return nil, errors.Wrap(err, "invalid Kibana endpoint") } clients[i] = &requestClient{ - request: prefixRequestFactory(kibanaURL), + request: prefixRequestFactory(url), client: httpClient, } } @@ -152,14 +148,14 @@ func NewWithConfig(log *logger.Logger, cfg *Config, wrapper wrapperFunc) (*Clien return new(log, cfg, clients...) } -// Send executes a direct calls against the Kibana API, the method will takes cares of cloning -// also add necessary headers for Kibana likes: "Content-Type", "Accept", and "kbn-xsrf". +// Send executes a direct calls against the API, the method will takes cares of cloning +// also add necessary headers for likes: "Content-Type", "Accept", and "kbn-xsrf". // No assumptions is done on the response concerning the received format, this will be the responsibility // of the implementation to correctly unpack any received data. // // NOTE: // - The caller of this method is free to override any value found in the headers. -// - The magic of unpack kibana errors is not done in the Send method, a helper method is provided. +// - The magic of unpacking of errors is not done in the Send method, a helper method is provided. func (c *Client) Send( ctx context.Context, method, path string, @@ -208,10 +204,10 @@ func (c *Client) URI() string { return string(c.config.Protocol) + "://" + host + "/" + c.config.Path } -// new creates new Kibana API client. +// new creates new API client. func new( log *logger.Logger, - cfg *Config, + cfg Config, httpClients ...*requestClient, ) (*Client, error) { c := &Client{ @@ -230,7 +226,7 @@ func (c *Client) nextRequester() *requestClient { now := time.Now().UTC() for _, requester := range c.clients { - if requester.lastErr != nil && now.Sub(requester.lastErrOcc) > kibanaRetryOnBadConnTimeout { + if requester.lastErr != nil && now.Sub(requester.lastErrOcc) > retryOnBadConnTimeout { requester.lastErr = nil requester.lastErrOcc = time.Time{} } diff --git a/x-pack/elastic-agent/pkg/kibana/client_test.go b/x-pack/elastic-agent/pkg/remote/client_test.go similarity index 85% rename from x-pack/elastic-agent/pkg/kibana/client_test.go rename to x-pack/elastic-agent/pkg/remote/client_test.go index 7dff58ffb6f8..ec304573f410 100644 --- a/x-pack/elastic-agent/pkg/kibana/client_test.go +++ b/x-pack/elastic-agent/pkg/remote/client_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package remote import ( "bytes" @@ -44,11 +44,11 @@ func TestPortDefaults(t *testing.T) { ExpectedPort int ExpectedScheme string }{ - {"no scheme uri", "test.url", kibanaPort, "http"}, - {"default kibana port", "http://test.url", kibanaPort, "http"}, - {"specified kibana port", "http://test.url:123", 123, "http"}, - {"default kibana https port", "https://test.url", kibanaHTTPSPort, "https"}, - {"specified kibana https port", "https://test.url:123", 123, "https"}, + {"no scheme uri", "test.url", defaultPort, "http"}, + {"default port", "http://test.url", defaultPort, "http"}, + {"specified port", "http://test.url:123", 123, "http"}, + {"default https port", "https://test.url", defaultPort, "https"}, + {"specified https port", "https://test.url:123", 123, "https"}, } for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { @@ -238,35 +238,6 @@ func TestHTTPClient(t *testing.T) { }, )) - t.Run("Enforce Kibana version", withServer( - func(t *testing.T) *http.ServeMux { - msg := `{ message: "hello" }` - mux := http.NewServeMux() - mux.HandleFunc("/echo-hello", enforceKibanaHandler(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, msg) - }, "8.0.0")) - return mux - }, func(t *testing.T, host string) { - cfg := config.MustNewConfigFrom(map[string]interface{}{ - "host": host, - }) - - client, err := NewWithRawConfig(nil, cfg, func(wrapped http.RoundTripper) (http.RoundTripper, error) { - return NewEnforceKibanaVersionRoundTripper(wrapped, "8.0.0"), nil - }) - - require.NoError(t, err) - resp, err := client.Send(ctx, "GET", "/echo-hello", nil, nil, nil) - require.NoError(t, err) - - body, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - defer resp.Body.Close() - assert.Equal(t, `{ message: "hello" }`, string(body)) - }, - )) - t.Run("Allows to debug HTTP request between a client and a server", withServer( func(t *testing.T) *http.ServeMux { msg := `{ "message": "hello" }` @@ -310,7 +281,7 @@ func TestNextRequester(t *testing.T) { t.Run("Picks first requester on initial call", func(t *testing.T) { one := &requestClient{} two := &requestClient{} - client, err := new(nil, nil, one, two) + client, err := new(nil, Config{}, one, two) require.NoError(t, err) assert.Equal(t, one, client.nextRequester()) }) @@ -321,7 +292,7 @@ func TestNextRequester(t *testing.T) { lastErrOcc: time.Now().UTC(), } two := &requestClient{} - client, err := new(nil, nil, one, two) + client, err := new(nil, Config{}, one, two) require.NoError(t, err) assert.Equal(t, two, client.nextRequester()) }) @@ -331,7 +302,7 @@ func TestNextRequester(t *testing.T) { lastUsed: time.Now().UTC(), } two := &requestClient{} - client, err := new(nil, nil, one, two) + client, err := new(nil, Config{}, one, two) require.NoError(t, err) assert.Equal(t, two, client.nextRequester()) }) @@ -346,7 +317,7 @@ func TestNextRequester(t *testing.T) { three := &requestClient{ lastUsed: time.Now().UTC().Add(-2 * time.Minute), } - client, err := new(nil, nil, one, two, three) + client, err := new(nil, Config{}, one, two, three) require.NoError(t, err) assert.Equal(t, two, client.nextRequester()) }) @@ -363,7 +334,7 @@ func TestNextRequester(t *testing.T) { three := &requestClient{ lastUsed: time.Now().UTC().Add(-2 * time.Minute), } - client, err := new(nil, nil, one, two, three) + client, err := new(nil, Config{}, one, two, three) require.NoError(t, err) assert.Equal(t, three, client.nextRequester()) }) @@ -384,7 +355,7 @@ func TestNextRequester(t *testing.T) { lastErr: fmt.Errorf("fake error"), lastErrOcc: time.Now().Add(-2 * time.Minute), } - client, err := new(nil, nil, one, two, three) + client, err := new(nil, Config{}, one, two, three) require.NoError(t, err) assert.Equal(t, two, client.nextRequester()) }) @@ -411,16 +382,6 @@ func basicAuthHandler(handler http.HandlerFunc, username, password, realm string } } -func enforceKibanaHandler(handler http.HandlerFunc, version string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("kbn-version") != version { - http.Error(w, "Bad Request", http.StatusBadRequest) - return - } - handler(w, r) - } -} - type debugStack struct { sync.Mutex messages []string diff --git a/x-pack/elastic-agent/pkg/kibana/config.go b/x-pack/elastic-agent/pkg/remote/config.go similarity index 85% rename from x-pack/elastic-agent/pkg/kibana/config.go rename to x-pack/elastic-agent/pkg/remote/config.go index 093390ae082e..02b32274f2fd 100644 --- a/x-pack/elastic-agent/pkg/kibana/config.go +++ b/x-pack/elastic-agent/pkg/remote/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package remote import ( "fmt" @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) -// Config is the configuration for the Kibana client. +// Config is the configuration for the client. type Config struct { Protocol Protocol `config:"protocol" yaml:"protocol"` SpaceID string `config:"space.id" yaml:"space.id,omitempty"` @@ -28,9 +28,9 @@ type Config struct { type Protocol string const ( - // ProtocolHTTP is HTTP protocol connection to Kibana. + // ProtocolHTTP is HTTP protocol connection. ProtocolHTTP Protocol = "http" - // ProtocolHTTPS is HTTPS protocol connection to Kibana. + // ProtocolHTTPS is HTTPS protocol connection. ProtocolHTTPS Protocol = "https" ) @@ -44,9 +44,9 @@ func (p *Protocol) Unpack(from string) error { return nil } -// DefaultClientConfig creates default configuration for kibana client. -func DefaultClientConfig() *Config { - return &Config{ +// DefaultClientConfig creates default configuration for client. +func DefaultClientConfig() Config { + return Config{ Protocol: ProtocolHTTP, Host: "localhost:5601", Path: "", @@ -64,7 +64,7 @@ func (c *Config) IsBasicAuth() bool { return len(c.Username) > 0 && len(c.Password) > 0 } -// GetHosts returns the hosts to connect to kibana. +// GetHosts returns the hosts to connect. // // This looks first at `Hosts` and then at `Host` when `Hosts` is not defined. func (c *Config) GetHosts() []string { diff --git a/x-pack/elastic-agent/pkg/kibana/config_test.go b/x-pack/elastic-agent/pkg/remote/config_test.go similarity index 98% rename from x-pack/elastic-agent/pkg/kibana/config_test.go rename to x-pack/elastic-agent/pkg/remote/config_test.go index 57809e747785..21843a4b0479 100644 --- a/x-pack/elastic-agent/pkg/kibana/config_test.go +++ b/x-pack/elastic-agent/pkg/remote/config_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package remote import ( "reflect" diff --git a/x-pack/elastic-agent/pkg/kibana/round_trippers.go b/x-pack/elastic-agent/pkg/remote/round_trippers.go similarity index 86% rename from x-pack/elastic-agent/pkg/kibana/round_trippers.go rename to x-pack/elastic-agent/pkg/remote/round_trippers.go index ded029356716..8c5b86f45ca2 100644 --- a/x-pack/elastic-agent/pkg/kibana/round_trippers.go +++ b/x-pack/elastic-agent/pkg/remote/round_trippers.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package remote import ( "bytes" @@ -123,26 +123,6 @@ func NewDebugRoundTripper(wrapped http.RoundTripper, log debugLogger) http.Round return &DebugRoundTripper{rt: wrapped, log: log} } -// EnforceKibanaVersionRoundTripper sets the kbn-version header on every request. -type EnforceKibanaVersionRoundTripper struct { - rt http.RoundTripper - version string -} - -// RoundTrip adds the kbn-version header, if the remote kibana is not equal or superior the call -/// will fail. -func (r *EnforceKibanaVersionRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - const key = "kbn-version" - req.Header.Set(key, r.version) - return r.rt.RoundTrip(req) -} - -// NewEnforceKibanaVersionRoundTripper enforce the remove endpoint to be a a certain version, if the -// remote kibana is not equal the call will fail. -func NewEnforceKibanaVersionRoundTripper(wrapped http.RoundTripper, version string) http.RoundTripper { - return &EnforceKibanaVersionRoundTripper{rt: wrapped, version: version} -} - // BasicAuthRoundTripper wraps any request using a basic auth. type BasicAuthRoundTripper struct { rt http.RoundTripper diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml index b45a3d35648d..28529004a2a3 100644 --- a/x-pack/elastic-agent/spec/endpoint.yml +++ b/x-pack/elastic-agent/spec/endpoint.yml @@ -48,17 +48,9 @@ rules: values: - true -- select_into: - selectors: [fleet.access_api_key, fleet.kibana] - path: fleet.api - - map: path: fleet rules: - - remove_key: - key: access_api_key - - remove_key: - key: kibana - remove_key: key: server diff --git a/x-pack/elastic-agent/spec/fleet-server.yml b/x-pack/elastic-agent/spec/fleet-server.yml index 50fded6ea55c..31cf9446da77 100644 --- a/x-pack/elastic-agent/spec/fleet-server.yml +++ b/x-pack/elastic-agent/spec/fleet-server.yml @@ -38,16 +38,10 @@ rules: - map: path: fleet rules: - - remove_key: - key: enabled - - remove_key: - key: access_api_key - - remove_key: - key: kibana - - remove_key: - key: reporting - - remove_key: - key: server + - filter: + selectors: + - agent + - host - map: path: inputs From 8b9d70bd8606c388a69bba331f325d6e5a276d99 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 8 Apr 2021 14:24:13 -0400 Subject: [PATCH 05/12] Disable banner logging from Azure EventHubs library (#24989) The filebeat log contains this when you use the `azure-eventhub` input. This disables that output. Apr 05 15:14:43 something filebeat[410843]: / ____/ _____ ____ / /_/ / / /_ __/ /_ _____ Apr 05 15:14:43 something filebeat[410843]: / __/ | | / / _ \/ __ \/ __/ /_/ / / / / __ \/ ___/ Apr 05 15:14:43 something filebeat[410843]: / /___ | |/ / __/ / / / /_/ __ / /_/ / /_/ (__ ) Apr 05 15:14:43 something filebeat[410843]: /_____/ |___/\___/_/ /_/\__/_/ /_/\__,_/_.___/____/ Apr 05 15:14:43 something filebeat[410843]: => processing events, ctrl+c to exit --- x-pack/filebeat/input/azureeventhub/eph.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index 7cc344faf28e..8fa976b63458 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -46,13 +46,15 @@ func (a *azureInput) runWithEPH() error { fmt.Sprintf("%s%s%s", a.config.ConnectionString, eventHubConnector, a.config.EventHubName), leaserCheckpointer, leaserCheckpointer, - eph.WithConsumerGroup(a.config.ConsumerGroup)) + eph.WithConsumerGroup(a.config.ConsumerGroup), + eph.WithNoBanner()) } else { a.processor, err = eph.NewFromConnectionString( a.workerCtx, fmt.Sprintf("%s%s%s", a.config.ConnectionString, eventHubConnector, a.config.EventHubName), leaserCheckpointer, - leaserCheckpointer) + leaserCheckpointer, + eph.WithNoBanner()) } if err != nil { return err From a94055b4096af5f4f8d44ac7714e4861d17302b4 Mon Sep 17 00:00:00 2001 From: Austin Songer Date: Thu, 8 Apr 2021 13:31:38 -0500 Subject: [PATCH 06/12] [Filebeat][Docs][7.12] Fix a Incorrect Spelling on Threat Intel Module (#24682) * Fix a Incorrect Spelling * Update CHANGELOG.next.asciidoc * Update CHANGELOG.next.asciidoc * Update CHANGELOG.next.asciidoc * Update docs.asciidoc * Update Filebeat-MISP-Overview.json * Update Filebeat-threatintel-misp.json --- filebeat/docs/modules/threatintel.asciidoc | 2 +- .../misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json | 2 +- x-pack/filebeat/module/threatintel/_meta/docs.asciidoc | 2 +- .../_meta/kibana/7/dashboard/Filebeat-threatintel-misp.json | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc index fa98046f2d52..f9efb4af87cc 100644 --- a/filebeat/docs/modules/threatintel.asciidoc +++ b/filebeat/docs/modules/threatintel.asciidoc @@ -382,7 +382,7 @@ Overview of the information provided by the Anomali Limo feed. image::./images/filebeat-threatintel-misp.png[] [float] -Overview of the information provided by the MSIP feed. +Overview of the information provided by the MISP feed. :modulename!: diff --git a/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json b/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json index 909548471d0d..6f8e08986d49 100644 --- a/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json +++ b/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json @@ -2,7 +2,7 @@ "objects": [ { "attributes": { - "description": "Overview dashboard for Filebeat MSIP module.", + "description": "Overview dashboard for Filebeat MISP module.", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": { diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc index 772bdda50b22..1be2a4fc8384 100644 --- a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc @@ -377,6 +377,6 @@ Overview of the information provided by the Anomali Limo feed. image::./images/filebeat-threatintel-misp.png[] [float] -Overview of the information provided by the MSIP feed. +Overview of the information provided by the MISP feed. :modulename!: diff --git a/x-pack/filebeat/module/threatintel/_meta/kibana/7/dashboard/Filebeat-threatintel-misp.json b/x-pack/filebeat/module/threatintel/_meta/kibana/7/dashboard/Filebeat-threatintel-misp.json index 8cf715e18e62..ca77b093cfd3 100644 --- a/x-pack/filebeat/module/threatintel/_meta/kibana/7/dashboard/Filebeat-threatintel-misp.json +++ b/x-pack/filebeat/module/threatintel/_meta/kibana/7/dashboard/Filebeat-threatintel-misp.json @@ -2,7 +2,7 @@ "objects": [ { "attributes": { - "description": "MSIP indicators ingested by the threat intel Filebeat module.", + "description": "MISP indicators ingested by the threat intel Filebeat module.", "hits": 0, "kibanaSavedObjectMeta": { "searchSourceJSON": { @@ -2096,4 +2096,4 @@ } ], "version": "7.11.1" -} \ No newline at end of file +} From a91bba523d2075272d0aad0bd5e7f006d29cdc84 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 8 Apr 2021 14:44:36 -0400 Subject: [PATCH 07/12] [Elastic Agent] Fix install with bootstrapping Fleet Server (#24981) * Fix install command to work with Fleet Server, remove the requirement for the enrollment token when bootstrapping with Fleet Server. * Fix tests. * Add changelog entry. * Simplify if statements. * Add installation success confirmation message. * Fix leaderelection provider. * Fix fleet decorator. --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + .../gateway/fleet/fleet_gateway.go | 8 +- .../gateway/fleet/noop_status_controller.go | 4 +- .../pkg/agent/application/info/agent_id.go | 10 +- .../pkg/agent/application/info/agent_info.go | 10 +- .../agent/application/info/agent_metadata.go | 2 +- .../agent/application/managed_mode_test.go | 2 +- .../handler_action_policy_change_test.go | 4 +- .../emitter/modifiers/fleet_decorator.go | 15 +-- .../modifiers/monitoring_decorator_test.go | 8 +- .../elastic-agent/pkg/agent/cmd/container.go | 21 ++-- .../elastic-agent/pkg/agent/cmd/enroll_cmd.go | 107 ++++++++++++++---- x-pack/elastic-agent/pkg/agent/cmd/inspect.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/install.go | 9 +- x-pack/elastic-agent/pkg/agent/cmd/run.go | 7 +- .../elastic-agent/pkg/agent/control/addr.go | 2 +- .../pkg/agent/control/server/server.go | 7 +- .../pkg/agent/install/uninstall.go | 2 +- .../pkg/agent/operation/common_test.go | 2 +- .../pkg/agent/operation/monitoring_test.go | 2 +- .../pkg/agent/operation/operator.go | 10 +- .../pkg/capabilities/capabilities.go | 2 +- .../elastic-agent/pkg/capabilities/input.go | 2 +- .../pkg/capabilities/input_test.go | 4 +- .../elastic-agent/pkg/capabilities/output.go | 2 +- .../elastic-agent/pkg/capabilities/upgrade.go | 2 +- .../pkg/composable/providers/agent/agent.go | 2 +- .../kubernetes_leaderelection.go | 2 +- .../pkg/core/monitoring/beats/monitoring.go | 6 +- .../pkg/core/plugin/process/app.go | 5 +- .../pkg/core/plugin/process/configure.go | 2 +- .../pkg/core/plugin/process/start.go | 2 +- .../pkg/core/plugin/service/app.go | 4 +- x-pack/elastic-agent/pkg/core/process/cmd.go | 10 +- .../pkg/core/process/cmd_darwin.go | 10 +- .../pkg/core/process/cmd_linux.go | 10 +- .../elastic-agent/pkg/core/process/process.go | 21 +++- .../elastic-agent/pkg/core/status/reporter.go | 13 ++- .../pkg/core/status/reporter_test.go | 36 +++--- 39 files changed, 241 insertions(+), 129 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 31db04aa7ef4..9af931816c93 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -48,6 +48,7 @@ - Fix nil pointer when null is generated as list item. {issue}23734[23734] - Add support for filestream input. {pull}24820[24820] - Add check for URL set when cert and cert key. {pull}24904[24904] +- Fix install command for Fleet Server bootstrap, remove need for --enrollment-token when using --fleet-server {pull}24981[24981] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go index 10c04ed60697..6018e2bc5f98 100644 --- a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go @@ -164,7 +164,7 @@ func (f *fleetGateway) worker() { resp, err := f.doExecute() if err != nil { f.log.Error(err) - f.statusReporter.Update(state.Failed, err.Error()) + f.statusReporter.Update(state.Failed, err.Error(), nil) continue } @@ -177,14 +177,14 @@ func (f *fleetGateway) worker() { if err := f.dispatcher.Dispatch(f.acker, actions...); err != nil { errMsg = fmt.Sprintf("failed to dispatch actions, error: %s", err) f.log.Error(errMsg) - f.statusReporter.Update(state.Failed, errMsg) + f.statusReporter.Update(state.Failed, errMsg, nil) } f.log.Debugf("FleetGateway is sleeping, next update in %s", f.settings.Duration) if errMsg != "" { - f.statusReporter.Update(state.Failed, errMsg) + f.statusReporter.Update(state.Failed, errMsg, nil) } else { - f.statusReporter.Update(state.Healthy, "") + f.statusReporter.Update(state.Healthy, "", nil) } case <-f.bgContext.Done(): diff --git a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/noop_status_controller.go b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/noop_status_controller.go index 59994d1d4546..620090321406 100644 --- a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/noop_status_controller.go +++ b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/noop_status_controller.go @@ -23,5 +23,5 @@ func (*noopController) StatusString() string { return type noopReporter struct{} -func (*noopReporter) Update(_ state.Status, _ string) {} -func (*noopReporter) Unregister() {} +func (*noopReporter) Update(_ state.Status, _ string, _ map[string]interface{}) {} +func (*noopReporter) Unregister() {} diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go index 393195830475..c560a7102715 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go @@ -38,7 +38,7 @@ type ioStore interface { // updateLogLevel updates log level and persists it to disk. func updateLogLevel(level string) error { - ai, err := loadAgentInfoWithBackoff(false, defaultLogLevel) + ai, err := loadAgentInfoWithBackoff(false, defaultLogLevel, false) if err != nil { return err } @@ -160,7 +160,7 @@ func yamlToReader(in interface{}) (io.Reader, error) { return bytes.NewReader(data), nil } -func loadAgentInfoWithBackoff(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) { +func loadAgentInfoWithBackoff(forceUpdate bool, logLevel string, createAgentID bool) (*persistentAgentInfo, error) { var err error var ai *persistentAgentInfo @@ -169,7 +169,7 @@ func loadAgentInfoWithBackoff(forceUpdate bool, logLevel string) (*persistentAge for i := 0; i <= maxRetriesloadAgentInfo; i++ { backExp.Wait() - ai, err = loadAgentInfo(forceUpdate, logLevel) + ai, err = loadAgentInfo(forceUpdate, logLevel, createAgentID) if err != filelock.ErrAppAlreadyRunning { break } @@ -179,7 +179,7 @@ func loadAgentInfoWithBackoff(forceUpdate bool, logLevel string) (*persistentAge return ai, err } -func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) { +func loadAgentInfo(forceUpdate bool, logLevel string, createAgentID bool) (*persistentAgentInfo, error) { idLock := paths.AgentConfigFileLock() if err := idLock.TryLock(); err != nil { return nil, err @@ -194,7 +194,7 @@ func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, err return nil, err } - if agentinfo != nil && !forceUpdate && agentinfo.ID != "" { + if agentinfo != nil && !forceUpdate && (agentinfo.ID != "" || !createAgentID) { return agentinfo, nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go index 313b894105be..ff0b1d8263bf 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go @@ -20,8 +20,8 @@ type AgentInfo struct { // new unique identifier for agent. // If agent config file does not exist it gets created. // Initiates log level to predefined value. -func NewAgentInfoWithLog(level string) (*AgentInfo, error) { - agentInfo, err := loadAgentInfoWithBackoff(false, level) +func NewAgentInfoWithLog(level string, createAgentID bool) (*AgentInfo, error) { + agentInfo, err := loadAgentInfoWithBackoff(false, level, createAgentID) if err != nil { return nil, err } @@ -37,8 +37,8 @@ func NewAgentInfoWithLog(level string) (*AgentInfo, error) { // this created ID otherwise it generates // new unique identifier for agent. // If agent config file does not exist it gets created. -func NewAgentInfo() (*AgentInfo, error) { - return NewAgentInfoWithLog(defaultLogLevel) +func NewAgentInfo(createAgentID bool) (*AgentInfo, error) { + return NewAgentInfoWithLog(defaultLogLevel, createAgentID) } // LogLevel updates log level of agent. @@ -53,7 +53,7 @@ func (i *AgentInfo) LogLevel(level string) error { // ReloadID reloads agent info ID from configuration file. func (i *AgentInfo) ReloadID() error { - newInfo, err := NewAgentInfoWithLog(i.logLevel) + newInfo, err := NewAgentInfoWithLog(i.logLevel, false) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go index ea44265141b7..c1b4f4055ee2 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -121,7 +121,7 @@ const ( // Metadata loads metadata from disk. func Metadata() (*ECSMeta, error) { - agentInfo, err := NewAgentInfo() + agentInfo, err := NewAgentInfo(false) if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index 177ab7cabf38..7d52d69d4b85 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -43,7 +43,7 @@ func TestManagedModeRouting(t *testing.T) { log, _ := logger.New("", false) router, _ := router.New(log, streamFn) - agentInfo, _ := info.NewAgentInfo() + agentInfo, _ := info.NewAgentInfo(true) nullStore := &storage.NullStore{} composableCtrl, _ := composable.New(log, nil) emit, err := emitter.New(ctx, log, agentInfo, composableCtrl, router, &pipeline.ConfigModifiers{Decorators: []pipeline.DecoratorFunc{modifiers.InjectMonitoring}}, nil) diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go index 1ab7ee20fdda..1c4c37509398 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change_test.go @@ -36,7 +36,7 @@ func (m *mockEmitter) Emitter(policy *config.Config) error { func TestPolicyChange(t *testing.T) { log, _ := logger.New("", false) ack := noopacker.NewAcker() - agentInfo, _ := info.NewAgentInfo() + agentInfo, _ := info.NewAgentInfo(true) nullStore := &storage.NullStore{} t.Run("Receive a config change and successfully emits a raw configuration", func(t *testing.T) { @@ -90,7 +90,7 @@ func TestPolicyChange(t *testing.T) { func TestPolicyAcked(t *testing.T) { log, _ := logger.New("", false) - agentInfo, _ := info.NewAgentInfo() + agentInfo, _ := info.NewAgentInfo(true) nullStore := &storage.NullStore{} t.Run("Config change should not ACK on error", func(t *testing.T) { diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go index 1d77e5022c59..4e1e71dd3bc1 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/fleet_decorator.go @@ -5,8 +5,6 @@ package modifiers import ( - "fmt" - "github.com/elastic/go-sysinfo/types" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" @@ -34,16 +32,15 @@ func InjectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag } fleet, ok := transpiler.Lookup(ast, "fleet") if !ok { - return fmt.Errorf("failed to get fleet from config") + // no fleet from configuration; skip + return nil } // copy top-level agent.* into fleet.agent.* (this gets sent to Applications in this structure) - agent, ok := transpiler.Lookup(ast, "agent") - if !ok { - return fmt.Errorf("failed to get agent key from config") - } - if err := transpiler.Insert(ast, agent, "fleet"); err != nil { - return err + if agent, ok := transpiler.Lookup(ast, "agent"); ok { + if err := transpiler.Insert(ast, agent, "fleet"); err != nil { + return err + } } // ensure that the agent.logging.level is present diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator_test.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator_test.go index 8b93fe762805..afb15edac80e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator_test.go @@ -14,7 +14,7 @@ import ( ) func TestMonitoringInjection(t *testing.T) { - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(true) if err != nil { t.Fatal(err) } @@ -93,7 +93,7 @@ GROUPLOOP: } func TestMonitoringInjectionDefaults(t *testing.T) { - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(true) if err != nil { t.Fatal(err) } @@ -172,7 +172,7 @@ GROUPLOOP: } func TestMonitoringInjectionDisabled(t *testing.T) { - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(true) if err != nil { t.Fatal(err) } @@ -261,7 +261,7 @@ GROUPLOOP: } func TestChangeInMonitoringWithChangeInInput(t *testing.T) { - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(true) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index b1902010ae3f..b926977f0810 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -237,15 +237,15 @@ func runContainerCmd(streams *cli.IOStreams, cmd *cobra.Command, cfg setupConfig } } if cfg.Fleet.Enroll { - if client == nil { - client, err = kibanaClient(cfg.Kibana) - if err != nil { - return err - } - } var policy *kibanaPolicy token := cfg.Fleet.EnrollmentToken - if token == "" { + if token == "" && !cfg.FleetServer.Enable { + if client == nil { + client, err = kibanaClient(cfg.Kibana) + if err != nil { + return err + } + } policy, err = kibanaFetchPolicy(client, cfg, streams) if err != nil { return err @@ -333,7 +333,10 @@ func buildEnrollArgs(cfg setupConfig, token string, policyID string) ([]string, args = append(args, "--certificate-authorities", cfg.Fleet.CA) } } - return append(args, "--enrollment-token", token), nil + if token != "" { + args = append(args, "--enrollment-token", token) + } + return args, nil } func buildFleetServerConnStr(cfg fleetServerConfig) (string, error) { @@ -572,7 +575,7 @@ func runLegacyAPMServer(streams *cli.IOStreams, path string) (*process.Info, err addEnv("--path.logs", "LOGS_PATH") addEnv("--httpprof", "HTTPPROF") logInfo(streams, "Starting legacy apm-server daemon as a subprocess.") - return process.Start(log, apmBinary, nil, os.Geteuid(), os.Getegid(), args...) + return process.Start(log, apmBinary, nil, os.Geteuid(), os.Getegid(), args) } func logToStderr(cfg *configuration.Configuration) { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go index fa8d10ad9034..96636822bf16 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go @@ -11,10 +11,9 @@ import ( "io" "math/rand" "os" + "os/exec" "time" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process" - "gopkg.in/yaml.v2" "github.com/elastic/beats/v7/libbeat/common/backoff" @@ -31,6 +30,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/authority" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" fleetclient "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/client" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" @@ -156,10 +156,13 @@ func (c *enrollCmd) Execute(ctx context.Context) error { var err error defer c.stopAgent() // ensure its stopped no matter what if c.options.FleetServer.ConnStr != "" { - err = c.fleetServerBootstrap(ctx) + token, err := c.fleetServerBootstrap(ctx) if err != nil { return err } + if c.options.EnrollAPIKey == "" && token != "" { + c.options.EnrollAPIKey = token + } } c.remoteConfig, err = c.options.remoteConfig() @@ -194,20 +197,25 @@ func (c *enrollCmd) Execute(ctx context.Context) error { return nil } -func (c *enrollCmd) fleetServerBootstrap(ctx context.Context) error { +func (c *enrollCmd) fleetServerBootstrap(ctx context.Context) (string, error) { c.log.Debug("verifying communication with running Elastic Agent daemon") agentRunning := true _, err := getDaemonStatus(ctx) if err != nil { if !c.options.FleetServer.SpawnAgent { - return errors.New("failed to communicate with elastic-agent daemon; is elastic-agent running?") + // wait longer to try and communicate with the Elastic Agent + err = waitForAgent(ctx) + if err != nil { + return "", errors.New("failed to communicate with elastic-agent daemon; is elastic-agent running?") + } + } else { + agentRunning = false } - agentRunning = false } err = c.prepareFleetTLS() if err != nil { - return err + return "", err } fleetConfig, err := createFleetServerBootstrapConfig( @@ -215,7 +223,7 @@ func (c *enrollCmd) fleetServerBootstrap(ctx context.Context) error { c.options.FleetServer.Host, c.options.FleetServer.Port, c.options.FleetServer.Cert, c.options.FleetServer.CertKey, c.options.FleetServer.ElasticsearchCA) if err != nil { - return err + return "", err } configToStore := map[string]interface{}{ @@ -223,11 +231,11 @@ func (c *enrollCmd) fleetServerBootstrap(ctx context.Context) error { } reader, err := yamlToReader(configToStore) if err != nil { - return err + return "", err } if err := safelyStoreAgentInfo(c.configStore, reader); err != nil { - return err + return "", err } var agentSubproc <-chan *os.ProcessState @@ -235,21 +243,21 @@ func (c *enrollCmd) fleetServerBootstrap(ctx context.Context) error { // reload the already running agent err = c.daemonReload(ctx) if err != nil { - return errors.New(err, "failed to trigger elastic-agent daemon reload", errors.TypeApplication) + return "", errors.New(err, "failed to trigger elastic-agent daemon reload", errors.TypeApplication) } } else { // spawn `run` as a subprocess so enroll can perform the bootstrap process of Fleet Server - agentSubproc, err = c.startAgent() + agentSubproc, err = c.startAgent(ctx) if err != nil { - return err + return "", err } } - err = waitForFleetServer(ctx, agentSubproc, c.log) + token, err := waitForFleetServer(ctx, agentSubproc, c.log) if err != nil { - return errors.New(err, "fleet-server never started by elastic-agent daemon", errors.TypeApplication) + return "", errors.New(err, "fleet-server never started by elastic-agent daemon", errors.TypeApplication) } - return nil + return token, nil } func (c *enrollCmd) prepareFleetTLS() error { @@ -421,22 +429,25 @@ func (c *enrollCmd) enroll(ctx context.Context) error { return nil } -func (c *enrollCmd) startAgent() (<-chan *os.ProcessState, error) { +func (c *enrollCmd) startAgent(ctx context.Context) (<-chan *os.ProcessState, error) { cmd, err := os.Executable() if err != nil { return nil, err } c.log.Info("Spawning Elastic Agent daemon as a subprocess to complete bootstrap process.") args := []string{ - "run", "-c", paths.ConfigFile(), + "run", "-e", "-c", paths.ConfigFile(), "--path.home", paths.Top(), "--path.config", paths.Config(), "--path.logs", paths.Logs(), } if !paths.IsVersionHome() { args = append(args, "--path.home.unversioned") } - proc, err := process.Start( - c.log, cmd, nil, os.Geteuid(), os.Getegid(), args...) + proc, err := process.StartContext( + ctx, c.log, cmd, nil, os.Geteuid(), os.Getegid(), args, func(c *exec.Cmd) { + c.Stdout = os.Stdout + c.Stderr = os.Stderr + }) if err != nil { return nil, err } @@ -486,10 +497,47 @@ func getDaemonStatus(ctx context.Context) (*client.AgentStatus, error) { } type waitResult struct { - err error + enrollmentToken string + err error +} + +func waitForAgent(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + resChan := make(chan waitResult) + innerCtx, innerCancel := context.WithCancel(context.Background()) + defer innerCancel() + go func() { + for { + <-time.After(1 * time.Second) + _, err := getDaemonStatus(innerCtx) + if err == context.Canceled { + resChan <- waitResult{err: err} + return + } + if err == nil { + resChan <- waitResult{} + break + } + } + }() + + var res waitResult + select { + case <-ctx.Done(): + innerCancel() + res = <-resChan + case res = <-resChan: + } + + if res.err != nil { + return res.err + } + return nil } -func waitForFleetServer(ctx context.Context, agentSubproc <-chan *os.ProcessState, log *logger.Logger) error { +func waitForFleetServer(ctx context.Context, agentSubproc <-chan *os.ProcessState, log *logger.Logger) (string, error) { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -544,7 +592,16 @@ func waitForFleetServer(ctx context.Context, agentSubproc <-chan *os.ProcessStat if app.Message != "" { log.Infof("Fleet Server - %s", app.Message) } - resChan <- waitResult{} + // extract the enrollment token from the status payload + token := "" + if app.Payload != nil { + if enrollToken, ok := app.Payload["enrollment_token"]; ok { + if tokenStr, ok := enrollToken.(string); ok { + token = tokenStr + } + } + } + resChan <- waitResult{enrollmentToken: token} break } else if app.Status == proto.Status_FAILED { // app completely failed; exit now @@ -591,9 +648,9 @@ func waitForFleetServer(ctx context.Context, agentSubproc <-chan *os.ProcessStat } if res.err != nil { - return res.err + return "", res.err } - return nil + return res.enrollmentToken, nil } func getAppFromStatus(status *client.AgentStatus, name string) *client.ApplicationStatus { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/inspect.go b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go index e66a15dfe3fc..f13163f27b29 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/inspect.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go @@ -63,7 +63,7 @@ func newInspectOutputCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra. outName, _ := c.Flags().GetString("output") program, _ := c.Flags().GetString("program") cfgPath := paths.ConfigFile() - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(false) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/install.go b/x-pack/elastic-agent/pkg/agent/cmd/install.go index 090f6a9f2feb..6df49f43ca65 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/install.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/install.go @@ -101,6 +101,10 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, args []string) error if url != "" && token != "" { askEnroll = false } + fleetServer, _ := cmd.Flags().GetString("fleet-server") + if fleetServer != "" { + askEnroll = false + } if force { askEnroll = false } @@ -114,12 +118,12 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, args []string) error enroll = false } } - if !askEnroll && (url == "" || token == "") { + if !askEnroll && (url == "" || token == "") && fleetServer == "" { // force was performed without required enrollment arguments, all done (standalone mode) enroll = false } - if enroll { + if enroll && fleetServer == "" { if url == "" { url, err = c.ReadInput("URL you want to enroll this Agent into:") if err != nil { @@ -187,5 +191,6 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, args []string) error } } + fmt.Fprint(streams.Out, "Elastic Agent has been successfully installed.\n") return nil } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index e4d5b1c105c4..75d9e49fad85 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -108,7 +108,12 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { // Windows: Mark override(cfg) } - agentInfo, err := info.NewAgentInfoWithLog(defaultLogLevel(cfg)) + // agent ID needs to stay empty in bootstrap mode + createAgentID := true + if cfg.Fleet != nil && cfg.Fleet.Server != nil && cfg.Fleet.Server.Bootstrap { + createAgentID = false + } + agentInfo, err := info.NewAgentInfoWithLog(defaultLogLevel(cfg), createAgentID) if err != nil { return errors.New(err, "could not load agent info", diff --git a/x-pack/elastic-agent/pkg/agent/control/addr.go b/x-pack/elastic-agent/pkg/agent/control/addr.go index fabaf4831403..7eca51789dbe 100644 --- a/x-pack/elastic-agent/pkg/agent/control/addr.go +++ b/x-pack/elastic-agent/pkg/agent/control/addr.go @@ -29,5 +29,5 @@ func Address() string { } // place in global /tmp to ensure that its small enough to fit; current path is way to long // for it to be used, but needs to be unique per Agent (in the case that multiple are running) - return fmt.Sprintf(`unix:///tmp/elastic-agent-%x.sock`, sha256.Sum256([]byte(path))) + return fmt.Sprintf(`unix:///tmp/elastic-agent/%x.sock`, sha256.Sum256([]byte(path))) } diff --git a/x-pack/elastic-agent/pkg/agent/control/server/server.go b/x-pack/elastic-agent/pkg/agent/control/server/server.go index edd96efdad64..25db5688c8e7 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/server.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/server.go @@ -6,6 +6,7 @@ package server import ( "context" + "encoding/json" "net" "sync" "time" @@ -175,12 +176,16 @@ func agentStatusToProto(code status.AgentStatusCode) proto.Status { func agentAppStatusToProto(apps []status.AgentApplicationStatus) []*proto.ApplicationStatus { s := make([]*proto.ApplicationStatus, len(apps)) for i, a := range apps { + var payload []byte + if a.Payload != nil { + payload, _ = json.Marshal(a.Payload) + } s[i] = &proto.ApplicationStatus{ Id: a.ID, Name: a.Name, Status: proto.Status(a.Status.ToProto()), Message: a.Message, - Payload: "", + Payload: string(payload), } } return s diff --git a/x-pack/elastic-agent/pkg/agent/install/uninstall.go b/x-pack/elastic-agent/pkg/agent/install/uninstall.go index 6a60a2c9258f..1524db81ca16 100644 --- a/x-pack/elastic-agent/pkg/agent/install/uninstall.go +++ b/x-pack/elastic-agent/pkg/agent/install/uninstall.go @@ -191,7 +191,7 @@ func programsFromConfig(cfg *config.Config) ([]program.Program, error) { return nil, errors.New("failed to create a ast from config", err) } - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(false) if err != nil { return nil, errors.New("failed to get an agent info", err) } diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 4eb24a105d39..505eadc8d082 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -50,7 +50,7 @@ func getTestOperator(t *testing.T, downloadPath string, installPath string, p *a } l := getLogger() - agentInfo, _ := info.NewAgentInfo() + agentInfo, _ := info.NewAgentInfo(true) fetcher := &DummyDownloader{} verifier := &DummyVerifier{} diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 6706e7094848..8a25eb809351 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -116,7 +116,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M } l := getLogger() - agentInfo, _ := info.NewAgentInfo() + agentInfo, _ := info.NewAgentInfo(true) fetcher := &DummyDownloader{} verifier := &DummyVerifier{} diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 35b9a6731b1c..63f1d3045668 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -146,7 +146,7 @@ func (o *Operator) Close() error { func (o *Operator) HandleConfig(cfg configrequest.Request) error { _, stateID, steps, ack, err := o.stateResolver.Resolve(cfg) if err != nil { - o.statusReporter.Update(state.Failed, err.Error()) + o.statusReporter.Update(state.Failed, err.Error(), nil) return errors.New(err, errors.TypeConfig, fmt.Sprintf("operator: failed to resolve configuration %s, error: %v", cfg, err)) } o.statusController.UpdateStateID(stateID) @@ -156,7 +156,7 @@ func (o *Operator) HandleConfig(cfg configrequest.Request) error { if _, isSupported := program.SupportedMap[strings.ToLower(step.ProgramSpec.Cmd)]; !isSupported { // mark failed, new config cannot be run msg := fmt.Sprintf("program '%s' is not supported", step.ProgramSpec.Cmd) - o.statusReporter.Update(state.Failed, msg) + o.statusReporter.Update(state.Failed, msg, nil) return errors.New(msg, errors.TypeApplication, errors.M(errors.MetaKeyAppName, step.ProgramSpec.Cmd)) @@ -166,19 +166,19 @@ func (o *Operator) HandleConfig(cfg configrequest.Request) error { handler, found := o.handlers[step.ID] if !found { msg := fmt.Sprintf("operator: received unexpected event '%s'", step.ID) - o.statusReporter.Update(state.Failed, msg) + o.statusReporter.Update(state.Failed, msg, nil) return errors.New(msg, errors.TypeConfig) } if err := handler(step); err != nil { msg := fmt.Sprintf("operator: failed to execute step %s, error: %v", step.ID, err) - o.statusReporter.Update(state.Failed, msg) + o.statusReporter.Update(state.Failed, msg, nil) return errors.New(err, errors.TypeConfig, msg) } } // Ack the resolver should state for next call. - o.statusReporter.Update(state.Healthy, "") + o.statusReporter.Update(state.Healthy, "", nil) ack() return nil diff --git a/x-pack/elastic-agent/pkg/capabilities/capabilities.go b/x-pack/elastic-agent/pkg/capabilities/capabilities.go index b03bef73d8c8..c0c012727525 100644 --- a/x-pack/elastic-agent/pkg/capabilities/capabilities.go +++ b/x-pack/elastic-agent/pkg/capabilities/capabilities.go @@ -87,7 +87,7 @@ func Load(capsFile string, log *logger.Logger, sc status.Controller) (Capability func (mgr *capabilitiesManager) Apply(in interface{}) (interface{}, error) { var err error // reset health on start, child caps will update to fail if needed - mgr.reporter.Update(state.Healthy, "") + mgr.reporter.Update(state.Healthy, "", nil) for _, cap := range mgr.caps { in, err = cap.Apply(in) if err != nil { diff --git a/x-pack/elastic-agent/pkg/capabilities/input.go b/x-pack/elastic-agent/pkg/capabilities/input.go index 11cb818883a2..3459846ec14a 100644 --- a/x-pack/elastic-agent/pkg/capabilities/input.go +++ b/x-pack/elastic-agent/pkg/capabilities/input.go @@ -166,7 +166,7 @@ func (c *inputCapability) renderInputs(inputs []map[string]interface{}) ([]map[s if !isSupported { msg := fmt.Sprintf("input '%s' is left out due to capability restriction '%s'", inputType, c.name()) c.log.Errorf(msg) - c.reporter.Update(state.Degraded, msg) + c.reporter.Update(state.Degraded, msg, nil) } newInputs = append(newInputs, input) diff --git a/x-pack/elastic-agent/pkg/capabilities/input_test.go b/x-pack/elastic-agent/pkg/capabilities/input_test.go index dd92f360c8a9..dbd32c6e9d17 100644 --- a/x-pack/elastic-agent/pkg/capabilities/input_test.go +++ b/x-pack/elastic-agent/pkg/capabilities/input_test.go @@ -395,5 +395,5 @@ func getInputsMap(tt ...string) map[string]interface{} { type testReporter struct{} -func (*testReporter) Update(state.Status, string) {} -func (*testReporter) Unregister() {} +func (*testReporter) Update(state.Status, string, map[string]interface{}) {} +func (*testReporter) Unregister() {} diff --git a/x-pack/elastic-agent/pkg/capabilities/output.go b/x-pack/elastic-agent/pkg/capabilities/output.go index 593b1bb31301..0b24c5838cf1 100644 --- a/x-pack/elastic-agent/pkg/capabilities/output.go +++ b/x-pack/elastic-agent/pkg/capabilities/output.go @@ -133,7 +133,7 @@ func (c *outputCapability) renderOutputs(outputs map[string]interface{}) (map[st if !isSupported { msg := fmt.Sprintf("output '%s' is left out due to capability restriction '%s'", outputName, c.name()) c.log.Errorf(msg) - c.reporter.Update(state.Degraded, msg) + c.reporter.Update(state.Degraded, msg, nil) } } diff --git a/x-pack/elastic-agent/pkg/capabilities/upgrade.go b/x-pack/elastic-agent/pkg/capabilities/upgrade.go index 94969e5dd40e..a98e56c149f2 100644 --- a/x-pack/elastic-agent/pkg/capabilities/upgrade.go +++ b/x-pack/elastic-agent/pkg/capabilities/upgrade.go @@ -129,7 +129,7 @@ func (c *upgradeCapability) Apply(upgradeMap map[string]interface{}) (map[string isSupported = !isSupported msg := fmt.Sprintf("upgrade is blocked out due to capability restriction '%s'", c.name()) c.log.Errorf(msg) - c.reporter.Update(state.Degraded, msg) + c.reporter.Update(state.Degraded, msg, nil) } if !isSupported { diff --git a/x-pack/elastic-agent/pkg/composable/providers/agent/agent.go b/x-pack/elastic-agent/pkg/composable/providers/agent/agent.go index efba2598ad0b..0214cb33abb8 100644 --- a/x-pack/elastic-agent/pkg/composable/providers/agent/agent.go +++ b/x-pack/elastic-agent/pkg/composable/providers/agent/agent.go @@ -22,7 +22,7 @@ type contextProvider struct{} // Run runs the Agent context provider. func (*contextProvider) Run(comm corecomp.ContextProviderComm) error { - a, err := info.NewAgentInfo() + a, err := info.NewAgentInfo(false) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go index acb5e732b824..5435936a46c7 100644 --- a/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go +++ b/x-pack/elastic-agent/pkg/composable/providers/kubernetesleaderelection/kubernetes_leaderelection.go @@ -56,7 +56,7 @@ func (p *contextProvider) Run(comm corecomp.ContextProviderComm) error { return nil } - agentInfo, err := info.NewAgentInfo() + agentInfo, err := info.NewAgentInfo(false) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go index 02b0c320d629..b10f0ef82a59 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go @@ -38,9 +38,9 @@ func getMonitoringEndpoint(spec program.Spec, operatingSystem, pipelineID string if len(path) < 104 { return path } - // place in global /tmp to ensure that its small enough to fit; current path is way to long + // place in global /tmp (or /var/tmp on Darwin) to ensure that its small enough to fit; current path is way to long // for it to be used, but needs to be unique per Agent (in the case that multiple are running) - return fmt.Sprintf(`unix:///tmp/elastic-agent-%x.sock`, sha256.Sum256([]byte(path))) + return fmt.Sprintf(`unix:///tmp/elastic-agent/%x.sock`, sha256.Sum256([]byte(path))) } func getLoggingFile(spec program.Spec, operatingSystem, installPath, pipelineID string) string { @@ -65,7 +65,7 @@ func AgentMonitoringEndpoint(operatingSystem string) string { } // place in global /tmp to ensure that its small enough to fit; current path is way to long // for it to be used, but needs to be unique per Agent (in the case that multiple are running) - return fmt.Sprintf(`unix:///tmp/elastic-agent-%x.sock`, sha256.Sum256([]byte(path))) + return fmt.Sprintf(`unix:///tmp/elastic-agent/%x.sock`, sha256.Sum256([]byte(path))) } // AgentPrefixedMonitoringEndpoint returns endpoint with exposed metrics for agent. diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 4586505db8d3..9f52f74ce38b 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -211,8 +211,7 @@ func (a *Application) watch(ctx context.Context, p app.Taggable, proc *process.I msg := fmt.Sprintf("exited with code: %d", procState.ExitCode()) a.setState(state.Crashed, msg, nil) - // it was a crash, cleanup anything required - go a.cleanUp() + // it was a crash a.start(ctx, p, cfg) a.appLock.Unlock() }() @@ -242,7 +241,7 @@ func (a *Application) setState(s state.Status, msg string, payload map[string]in if a.reporter != nil { go a.reporter.OnStateChange(a.id, a.name, a.state) } - a.statusReporter.Update(s, msg) + a.statusReporter.Update(s, msg, payload) } } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/configure.go b/x-pack/elastic-agent/pkg/core/plugin/process/configure.go index 23ebdafbf60d..a3f96458b6c9 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/configure.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/configure.go @@ -19,7 +19,7 @@ func (a *Application) Configure(_ context.Context, config map[string]interface{} if err != nil { // inject App metadata err = errors.New(err, errors.M(errors.MetaKeyAppName, a.name), errors.M(errors.MetaKeyAppName, a.id)) - a.statusReporter.Update(state.Degraded, err.Error()) + a.statusReporter.Update(state.Degraded, err.Error(), nil) } }() diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index 2a04bd8b6566..204c3747a99c 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -112,7 +112,7 @@ func (a *Application) start(ctx context.Context, t app.Taggable, cfg map[string] a.processConfig, a.uid, a.gid, - spec.Args...) + spec.Args) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index b33561f305f5..3cb14dc46c61 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -208,7 +208,7 @@ func (a *Application) Configure(_ context.Context, config map[string]interface{} if err != nil { // inject App metadata err = errors.New(err, errors.M(errors.MetaKeyAppName, a.name), errors.M(errors.MetaKeyAppName, a.id)) - a.statusReporter.Update(state.Degraded, err.Error()) + a.statusReporter.Update(state.Degraded, err.Error(), nil) } }() @@ -297,7 +297,7 @@ func (a *Application) setState(s state.Status, msg string, payload map[string]in if a.reporter != nil { go a.reporter.OnStateChange(a.id, a.name, a.state) } - a.statusReporter.Update(s, msg) + a.statusReporter.Update(s, msg, payload) } } diff --git a/x-pack/elastic-agent/pkg/core/process/cmd.go b/x-pack/elastic-agent/pkg/core/process/cmd.go index 3e4bc5da7d1a..c9d93df4047a 100644 --- a/x-pack/elastic-agent/pkg/core/process/cmd.go +++ b/x-pack/elastic-agent/pkg/core/process/cmd.go @@ -8,6 +8,7 @@ package process import ( + "context" "os" "os/exec" "path/filepath" @@ -15,8 +16,13 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -func getCmd(logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { - cmd := exec.Command(path, arg...) +func getCmd(ctx context.Context, logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { + var cmd *exec.Cmd + if ctx == nil { + cmd = exec.Command(path, arg...) + } else { + cmd = exec.CommandContext(ctx, path, arg...) + } cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, env...) cmd.Dir = filepath.Dir(path) diff --git a/x-pack/elastic-agent/pkg/core/process/cmd_darwin.go b/x-pack/elastic-agent/pkg/core/process/cmd_darwin.go index 6f17b61c800d..73ad4932d69f 100644 --- a/x-pack/elastic-agent/pkg/core/process/cmd_darwin.go +++ b/x-pack/elastic-agent/pkg/core/process/cmd_darwin.go @@ -7,6 +7,7 @@ package process import ( + "context" "math" "os" "os/exec" @@ -16,8 +17,13 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -func getCmd(logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { - cmd := exec.Command(path, arg...) +func getCmd(ctx context.Context, logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { + var cmd *exec.Cmd + if ctx == nil { + cmd = exec.Command(path, arg...) + } else { + cmd = exec.CommandContext(ctx, path, arg...) + } cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, env...) cmd.Dir = filepath.Dir(path) diff --git a/x-pack/elastic-agent/pkg/core/process/cmd_linux.go b/x-pack/elastic-agent/pkg/core/process/cmd_linux.go index d580dc80beda..ce6762713c0e 100644 --- a/x-pack/elastic-agent/pkg/core/process/cmd_linux.go +++ b/x-pack/elastic-agent/pkg/core/process/cmd_linux.go @@ -7,6 +7,7 @@ package process import ( + "context" "math" "os" "os/exec" @@ -16,8 +17,13 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -func getCmd(logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { - cmd := exec.Command(path, arg...) +func getCmd(ctx context.Context, logger *logger.Logger, path string, env []string, uid, gid int, arg ...string) *exec.Cmd { + var cmd *exec.Cmd + if ctx == nil { + cmd = exec.Command(path, arg...) + } else { + cmd = exec.CommandContext(ctx, path, arg...) + } cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, env...) cmd.Dir = filepath.Dir(path) diff --git a/x-pack/elastic-agent/pkg/core/process/process.go b/x-pack/elastic-agent/pkg/core/process/process.go index 9dcb86ae620e..70255a1ac5dd 100644 --- a/x-pack/elastic-agent/pkg/core/process/process.go +++ b/x-pack/elastic-agent/pkg/core/process/process.go @@ -5,9 +5,11 @@ package process import ( + "context" "fmt" "io" "os" + "os/exec" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" @@ -25,13 +27,28 @@ type Info struct { Stdin io.WriteCloser } +// Option is an option func to change the underlying command +type Option func(c *exec.Cmd) + // Start starts a new process // Returns: // - network address of child process // - process id // - error -func Start(logger *logger.Logger, path string, config *Config, uid, gid int, arg ...string) (proc *Info, err error) { - cmd := getCmd(logger, path, []string{}, uid, gid, arg...) +func Start(logger *logger.Logger, path string, config *Config, uid, gid int, args []string, opts ...Option) (proc *Info, err error) { + return StartContext(nil, logger, path, config, uid, gid, args, opts...) +} + +// StartContext starts a new process with context. +// Returns: +// - network address of child process +// - process id +// - error +func StartContext(ctx context.Context, logger *logger.Logger, path string, config *Config, uid, gid int, args []string, opts ...Option) (proc *Info, err error) { + cmd := getCmd(ctx, logger, path, []string{}, uid, gid, args...) + for _, o := range opts { + o(cmd) + } stdin, err := cmd.StdinPipe() if err != nil { return nil, err diff --git a/x-pack/elastic-agent/pkg/core/status/reporter.go b/x-pack/elastic-agent/pkg/core/status/reporter.go index d4abd96a990f..e986a0a43cf7 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter.go @@ -5,6 +5,7 @@ package status import ( + "reflect" "sync" "github.com/google/uuid" @@ -38,6 +39,7 @@ type AgentApplicationStatus struct { Name string Status state.Status Message string + Payload map[string]interface{} } // AgentStatus returns the overall status of the Elastic Agent. @@ -167,6 +169,7 @@ func (r *controller) Status() AgentStatus { Name: rep.name, Status: rep.status, Message: rep.message, + Payload: rep.payload, }) rep.mx.Unlock() } @@ -240,7 +243,7 @@ func (r *controller) StatusString() string { // Reporter reports status of component type Reporter interface { - Update(state.Status, string) + Update(state.Status, string, map[string]interface{}) Unregister() } @@ -251,21 +254,23 @@ type reporter struct { isRegistered bool status state.Status message string + payload map[string]interface{} unregisterFunc func() notifyChangeFunc func() } // Update updates the status of a component. -func (r *reporter) Update(s state.Status, message string) { +func (r *reporter) Update(s state.Status, message string, payload map[string]interface{}) { r.mx.Lock() defer r.mx.Unlock() if !r.isRegistered { return } - r.message = message - if r.status != s { + if r.status != s || r.message != message || !reflect.DeepEqual(r.payload, payload) { r.status = s + r.message = message + r.payload = payload r.notifyChangeFunc() } } diff --git a/x-pack/elastic-agent/pkg/core/status/reporter_test.go b/x-pack/elastic-agent/pkg/core/status/reporter_test.go index bc601155a43a..5facaeb8caa9 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter_test.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter_test.go @@ -30,12 +30,12 @@ func TestReporter(t *testing.T) { a2 := r.RegisterApp("app-2", "app") a3 := r.RegisterApp("other-1", "other") - r1.Update(state.Healthy, "") - r2.Update(state.Healthy, "") - r3.Update(state.Healthy, "") - a1.Update(state.Healthy, "") - a2.Update(state.Healthy, "") - a3.Update(state.Healthy, "") + r1.Update(state.Healthy, "", nil) + r2.Update(state.Healthy, "", nil) + r3.Update(state.Healthy, "", nil) + a1.Update(state.Healthy, "", nil) + a2.Update(state.Healthy, "", nil) + a3.Update(state.Healthy, "", nil) assert.Equal(t, Healthy, r.StatusCode()) assert.Equal(t, "online", r.StatusString()) @@ -47,9 +47,9 @@ func TestReporter(t *testing.T) { r2 := r.RegisterComponent("r2") r3 := r.RegisterComponent("r3") - r1.Update(state.Healthy, "") - r2.Update(state.Degraded, "degraded") - r3.Update(state.Healthy, "") + r1.Update(state.Healthy, "", nil) + r2.Update(state.Degraded, "degraded", nil) + r3.Update(state.Healthy, "", nil) assert.Equal(t, Degraded, r.StatusCode()) assert.Equal(t, "degraded", r.StatusString()) @@ -61,9 +61,9 @@ func TestReporter(t *testing.T) { r2 := r.RegisterComponent("r2") r3 := r.RegisterComponent("r3") - r1.Update(state.Healthy, "") - r2.Update(state.Failed, "failed") - r3.Update(state.Healthy, "") + r1.Update(state.Healthy, "", nil) + r2.Update(state.Failed, "failed", nil) + r3.Update(state.Healthy, "", nil) assert.Equal(t, Failed, r.StatusCode()) assert.Equal(t, "error", r.StatusString()) @@ -75,9 +75,9 @@ func TestReporter(t *testing.T) { r2 := r.RegisterComponent("r2") r3 := r.RegisterComponent("r3") - r1.Update(state.Healthy, "") - r2.Update(state.Failed, "failed") - r3.Update(state.Degraded, "degraded") + r1.Update(state.Healthy, "", nil) + r2.Update(state.Failed, "failed", nil) + r3.Update(state.Degraded, "degraded", nil) assert.Equal(t, Failed, r.StatusCode()) assert.Equal(t, "error", r.StatusString()) @@ -89,9 +89,9 @@ func TestReporter(t *testing.T) { r2 := r.RegisterComponent("r2") r3 := r.RegisterComponent("r3") - r1.Update(state.Healthy, "") - r2.Update(state.Failed, "failed") - r3.Update(state.Degraded, "degraded") + r1.Update(state.Healthy, "", nil) + r2.Update(state.Failed, "failed", nil) + r3.Update(state.Degraded, "degraded", nil) r2.Unregister() From 23e4403ae093fcc8f7905345cad2c7ad256976d8 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Thu, 8 Apr 2021 15:13:12 -0400 Subject: [PATCH 08/12] Add http.request.id to nginx/ingress_controller and elasticsearch/audit (#24994) * Add http.request.id to nginx/ingress_controller and elasticsearch/audit * Add changelog entry --- CHANGELOG.next.asciidoc | 1 + .../audit/ingest/pipeline-json.yml | 363 ++++++------- .../test/test-audit-711.log-expected.json | 3 + .../test/test-audit-730.log-expected.json | 10 + .../test/test-audit-761.log-expected.json | 1 + .../test/test-audit-docker.log-expected.json | 2 + .../audit/test/test-audit.log-expected.json | 8 + .../ingress_controller/ingest/pipeline.yml | 498 +++++++++--------- .../test/test.log-expected.json | 23 + 9 files changed, 485 insertions(+), 424 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 93bae39ef96c..e65221b62947 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -819,6 +819,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for upper case field names in Sophos XG module {pull}24693[24693] - Add `fail_on_template_error` option for httpjson input. {pull}24784[24784] - Change `okta.target` to `flattened` field type. {issue}24354[24354] {pull}24636[24636] +- Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml index 047942ef960d..e658a2836e0a 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -1,181 +1,188 @@ description: Pipeline for parsing elasticsearch audit logs in JSON format processors: -- json: - field: message - target_field: elasticsearch.audit -- remove: - field: elasticsearch.audit.type - ignore_missing: true -- date: - if: ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null - field: elasticsearch.audit.@timestamp - target_field: elasticsearch.audit.@timestamp - formats: - - yyyy-MM-dd'T'HH:mm:ss,SSS - - yyyy-MM-dd'T'HH:mm:ss,SSSZ - timezone: '{{ event.timezone }}' - ignore_failure: true -- remove: - if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null - field: event.timezone -- rename: - field: elasticsearch.audit.timestamp - target_field: elasticsearch.audit.@timestamp - ignore_missing: true -- dot_expander: - field: event.action - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.event.action - target_field: event.action - ignore_missing: true -- dot_expander: - field: event.type - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.event.type - target_field: elasticsearch.audit.layer - ignore_missing: true -- dot_expander: - field: origin.address - path: elasticsearch.audit -- grok: - field: elasticsearch.audit.origin.address - patterns: - - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int} - - '%{IPORHOST:source.ip}:%{INT:source.port:int}' - ignore_missing: true -- rename: - field: elasticsearch.audit.origin.address - target_field: source.address - ignore_missing: true -- dot_expander: - field: url.path - path: elasticsearch.audit -- dot_expander: - field: url.query - path: elasticsearch.audit -- set: - if: ctx.elasticsearch.audit?.url?.query == null - field: url.original - value: '{{elasticsearch.audit.url.path}}' - ignore_empty_value: true -- set: - if: ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null - field: url.original - value: '{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}' -- remove: - if: ctx.elasticsearch.audit?.url?.path != null - field: elasticsearch.audit.url.path -- remove: - if: ctx.elasticsearch.audit?.url?.query != null - field: elasticsearch.audit.url.query -- dot_expander: - field: node.id - path: elasticsearch.audit -- dot_expander: - field: node.name - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.node - target_field: elasticsearch.node -- rename: - field: elasticsearch.audit.change.disable.user.name - target_field: user.name - ignore_missing: true -- rename: - field: elasticsearch.audit.change.enable.user.name - target_field: user.name - ignore_missing: true -- rename: - field: elasticsearch.audit.delete.user.name - target_field: user.name - ignore_missing: true -- rename: - field: elasticsearch.audit.put.user.name - target_field: user.name - ignore_missing: true -- rename: - field: elasticsearch.audit.put.user.full_name - target_field: user.full_name - ignore_missing: true -- rename: - field: elasticsearch.audit.put.user.email - target_field: user.email - ignore_missing: true -- remove: - field: elasticsearch.audit.put - ignore_missing: true -- rename: - field: elasticsearch.audit.invalidate.apikeys.user.name - target_field: user.name - ignore_missing: true -- rename: - field: elasticsearch.audit.invalidate.apikeys.user.realm - target_field: elasticsearch.audit.user.realm - ignore_missing: true -- dot_expander: - field: user.run_as.name - path: elasticsearch.audit - ignore_failure: true -- dot_expander: - field: user.run_as.realm - path: elasticsearch.audit - ignore_failure: true -- convert: - field: elasticsearch.audit.user.run_as.name - target_field: user.effective.name - type: string - ignore_failure: true -- dot_expander: - field: user.name - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.user.name - target_field: user.name - ignore_missing: true -- dot_expander: - field: user.email - path: elasticsearch.audit -- dot_expander: - field: request.method - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.request.method - target_field: http.request.method - ignore_missing: true -- dot_expander: - field: request.body - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.request.body - target_field: http.request.body.content - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.audit -- dot_expander: - field: cluster.uuid - path: elasticsearch.audit -- rename: - field: elasticsearch.audit.cluster.name - target_field: elasticsearch.cluster.name - ignore_missing: true -- rename: - field: elasticsearch.audit.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- rename: - field: elasticsearch.audit.level - target_field: log.level - ignore_missing: true -- date: - field: elasticsearch.audit.@timestamp - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true + - json: + field: message + target_field: elasticsearch.audit + - remove: + field: elasticsearch.audit.type + ignore_missing: true + - date: + if: ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null + field: elasticsearch.audit.@timestamp + target_field: elasticsearch.audit.@timestamp + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + - yyyy-MM-dd'T'HH:mm:ss,SSSZ + timezone: "{{ event.timezone }}" + ignore_failure: true + - remove: + if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null + field: event.timezone + - rename: + field: elasticsearch.audit.timestamp + target_field: elasticsearch.audit.@timestamp + ignore_missing: true + - dot_expander: + field: event.action + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.event.action + target_field: event.action + ignore_missing: true + - dot_expander: + field: event.type + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.event.type + target_field: elasticsearch.audit.layer + ignore_missing: true + - dot_expander: + field: origin.address + path: elasticsearch.audit + - grok: + field: elasticsearch.audit.origin.address + patterns: + - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int} + - "%{IPORHOST:source.ip}:%{INT:source.port:int}" + ignore_missing: true + - rename: + field: elasticsearch.audit.origin.address + target_field: source.address + ignore_missing: true + - dot_expander: + field: url.path + path: elasticsearch.audit + - dot_expander: + field: url.query + path: elasticsearch.audit + - set: + if: ctx.elasticsearch.audit?.url?.query == null + field: url.original + value: "{{elasticsearch.audit.url.path}}" + ignore_empty_value: true + - set: + if: ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null + field: url.original + value: "{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}" + - remove: + if: ctx.elasticsearch.audit?.url?.path != null + field: elasticsearch.audit.url.path + - remove: + if: ctx.elasticsearch.audit?.url?.query != null + field: elasticsearch.audit.url.query + - dot_expander: + field: node.id + path: elasticsearch.audit + - dot_expander: + field: node.name + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.node + target_field: elasticsearch.node + - rename: + field: elasticsearch.audit.change.disable.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.change.enable.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.delete.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.full_name + target_field: user.full_name + ignore_missing: true + - rename: + field: elasticsearch.audit.put.user.email + target_field: user.email + ignore_missing: true + - remove: + field: elasticsearch.audit.put + ignore_missing: true + - rename: + field: elasticsearch.audit.invalidate.apikeys.user.name + target_field: user.name + ignore_missing: true + - rename: + field: elasticsearch.audit.invalidate.apikeys.user.realm + target_field: elasticsearch.audit.user.realm + ignore_missing: true + - dot_expander: + field: user.run_as.name + path: elasticsearch.audit + ignore_failure: true + - dot_expander: + field: user.run_as.realm + path: elasticsearch.audit + ignore_failure: true + - convert: + field: elasticsearch.audit.user.run_as.name + target_field: user.effective.name + type: string + ignore_failure: true + - dot_expander: + field: user.name + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.user.name + target_field: user.name + ignore_missing: true + - dot_expander: + field: user.email + path: elasticsearch.audit + - dot_expander: + field: request.method + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.request.method + target_field: http.request.method + ignore_missing: true + - dot_expander: + field: request.body + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.request.body + target_field: http.request.body.content + ignore_missing: true + - dot_expander: + field: request.id + path: elasticsearch.audit + - set: + field: http.request.id + copy_from: elasticsearch.audit.request.id + ignore_empty_value: true + - dot_expander: + field: cluster.name + path: elasticsearch.audit + - dot_expander: + field: cluster.uuid + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.cluster.name + target_field: elasticsearch.cluster.name + ignore_missing: true + - rename: + field: elasticsearch.audit.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true + - rename: + field: elasticsearch.audit.level + target_field: log.level + ignore_missing: true + - date: + field: elasticsearch.audit.@timestamp + target_field: "@timestamp" + formats: + - ISO8601 + ignore_failure: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json index 161d7542b041..fc9cbb5336b2 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json @@ -17,6 +17,7 @@ "event.timezone": "-02:00", "fileset.name": "audit", "host.id": "UwRu4mReRtyJO1-FWAPvIQ", + "http.request.id": "474ZciqtQteOhjLO3OdZIw", "input.type": "log", "log.offset": 0, "message": "{\"@timestamp\":\"2019-09-05T14:02:37,921\", \"node.id\":\"UwRu4mReRtyJO1-FWAPvIQ\", \"event.type\":\"transport\", \"event.action\":\"authentication_success\", \"user.name\":\"_system\", \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"realm\":\"__fallback\", \"request.id\":\"474ZciqtQteOhjLO3OdZIw\", \"action\":\"indices:monitor/stats\", \"request.name\":\"IndicesStatsRequest\"}", @@ -50,6 +51,7 @@ "event.timezone": "-02:00", "fileset.name": "audit", "host.id": "DJKjhISiTzy-JY5nCU8h3Q", + "http.request.id": "I9bQCw28Qfe4HWtIJHgoAg", "input.type": "log", "log.offset": 363, "message": "{\"@timestamp\":\"2020-01-29T09:41:10,856\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}", @@ -83,6 +85,7 @@ "event.timezone": "-02:00", "fileset.name": "audit", "host.id": "DJKjhISiTzy-JY5nCU8h3Q", + "http.request.id": "I9bQCw28Qfe4HWtIJHgoAg", "input.type": "log", "log.offset": 785, "message": "{\"@timestamp\":\"2020-01-29T09:41:10,859\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json index 0ee154a4023a..b41abec2b696 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json @@ -23,6 +23,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "qxwx-kV9Q9uSwmaeQp1OgQ", "input.type": "log", "log.offset": 0, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:08,484-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53568\", \"request.id\":\"qxwx-kV9Q9uSwmaeQp1OgQ\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", @@ -59,6 +60,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "YhpBByKQTvSJfqCmMyWtXg", "input.type": "log", "log.offset": 423, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:08,484-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53566\", \"request.id\":\"YhpBByKQTvSJfqCmMyWtXg\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", @@ -95,6 +97,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "h50kZy1fTM6F3uP7MyFPDw", "input.type": "log", "log.offset": 846, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,084-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53562\", \"request.id\":\"h50kZy1fTM6F3uP7MyFPDw\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", @@ -130,6 +133,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "7KZfVjrYToq8LGLW5tcyDA", "input.type": "log", "log.offset": 1269, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,611-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53570\", \"request.id\":\"7KZfVjrYToq8LGLW5tcyDA\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\".kibana_task_manager\"]}", @@ -165,6 +169,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "7KZfVjrYToq8LGLW5tcyDA", "input.type": "log", "log.offset": 1706, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,612-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53570\", \"request.id\":\"7KZfVjrYToq8LGLW5tcyDA\", \"action\":\"indices:data/read/search[phase/query]\", \"request.name\":\"ShardSearchTransportRequest\", \"indices\":[\".kibana_task_manager\"]}", @@ -197,6 +202,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "TAklb9dXRtSg5V9fybe2Kw", "input.type": "log", "log.offset": 2170, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,758-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53572\", \"request.id\":\"TAklb9dXRtSg5V9fybe2Kw\", \"action\":\"cluster:monitor/nodes/info\", \"request.name\":\"NodesInfoRequest\"}", @@ -229,6 +235,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "TAklb9dXRtSg5V9fybe2Kw", "input.type": "log", "log.offset": 2576, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,758-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53572\", \"request.id\":\"TAklb9dXRtSg5V9fybe2Kw\", \"action\":\"cluster:monitor/nodes/info[n]\", \"request.name\":\"NodeInfoRequest\"}", @@ -264,6 +271,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "7mgKUfdjQhmlrw7ZDG1FjQ", "input.type": "log", "log.offset": 2984, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,366-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53571\", \"request.id\":\"7mgKUfdjQhmlrw7ZDG1FjQ\", \"action\":\"indices:data/read/get\", \"request.name\":\"GetRequest\", \"indices\":[\".kibana\"]}", @@ -299,6 +307,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "7mgKUfdjQhmlrw7ZDG1FjQ", "input.type": "log", "log.offset": 3402, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,372-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53571\", \"request.id\":\"7mgKUfdjQhmlrw7ZDG1FjQ\", \"action\":\"indices:data/read/get[s]\", \"request.name\":\"GetRequest\", \"indices\":[\".kibana\"]}", @@ -331,6 +340,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "MA2xjPZLSvmif8VZ86OJZw", + "http.request.id": "7VcDBbunQnqT2_cGxtURXA", "input.type": "log", "log.offset": 3823, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,381-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53574\", \"request.id\":\"7VcDBbunQnqT2_cGxtURXA\", \"action\":\"cluster:admin/xpack/monitoring/bulk\", \"request.name\":\"MonitoringBulkRequest\"}", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json index c2bb04680654..5419e2e63eaa 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json @@ -31,6 +31,7 @@ "event.timezone": "-02:00", "fileset.name": "audit", "host.id": "vvj136QVQ2Ci2aXmrhyi3Q", + "http.request.id": "rLBMfPM2Q9q-DQEB_g30ww", "input.type": "log", "log.offset": 0, "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json index 66f14a2381ca..ff553822c752 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json @@ -13,6 +13,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", + "http.request.id": "pkduyMB5Tly6xgmkYbZi-A", "http.request.method": "GET", "input.type": "log", "log.offset": 0, @@ -58,6 +59,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", + "http.request.id": "KPgEINaXSbGNaIobp8OcMw", "http.request.method": "GET", "input.type": "log", "log.offset": 690, diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json index ce89459bd51f..cfd361072cf1 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json @@ -202,6 +202,7 @@ "fileset.name": "audit", "host.id": "y8fa3M5zSSGo1M_KJRMUXw", "http.request.body.content": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n", + "http.request.id": "WzL_kb6VSvOhAq0twPvHOQ", "http.request.method": "GET", "http.request.mime_type": "application/json", "input.type": "log", @@ -230,6 +231,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "qvLIgw_eTvyK3cgV-GaLVg", "input.type": "log", "log.offset": 2509, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T23:17:28,308+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"security_config_change\", \"event.action\":\"change_disable_user\", \"request.id\":\"qvLIgw_eTvyK3cgV-GaLVg\", \"change\":{\"disable\":{\"user\":{\"name\":\"user1\"}}}}", @@ -252,6 +254,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "BO3QU3qeTb-Ei0G0rUOalQ", "input.type": "log", "log.offset": 2770, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T23:17:34,843+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"security_config_change\", \"event.action\":\"change_enable_user\", \"request.id\":\"BO3QU3qeTb-Ei0G0rUOalQ\", \"change\":{\"enable\":{\"user\":{\"name\":\"user1\"}}}}", @@ -274,6 +277,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "au5a1Cc3RrebDMitMGGNCw", "input.type": "log", "log.offset": 3029, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T22:19:41,345+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"security_config_change\", \"event.action\":\"delete_user\", \"request.id\":\"au5a1Cc3RrebDMitMGGNCw\", \"delete\":{\"user\":{\"name\":\"jacknich\"}}}", @@ -298,6 +302,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "9clhpgjJRR-iKzOw20xBNQ", + "http.request.id": "7lyIQU9QTFqSrTxD0CqnTQ", "input.type": "log", "log.offset": 3273, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-31T00:36:30,247+0200\", \"node.id\":\"9clhpgjJRR-iKzOw20xBNQ\", \"event.type\":\"security_config_change\", \"event.action\":\"invalidate_apikeys\", \"request.id\":\"7lyIQU9QTFqSrTxD0CqnTQ\", \"invalidate\":{\"apikeys\":{\"owned_by_authenticated_user\":false,\"user\":{\"name\":\"myuser\",\"realm\":\"native1\"}}}}", @@ -320,6 +325,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "VIiSvhp4Riim_tpkQCVSQA", "input.type": "log", "log.offset": 3592, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T22:10:09,749+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"security_config_change\", \"event.action\":\"put_user\", \"request.id\":\"VIiSvhp4Riim_tpkQCVSQA\", \"put\":{\"user\":{\"name\":\"user1\",\"enabled\":false,\"roles\":[\"admin\",\"other_role1\"],\"full_name\":\"Jack Sparrow\",\"email\":\"jack@blackpearl.com\",\"has_password\":true,\"metadata\":{\"cunning\":10}}}}", @@ -356,6 +362,7 @@ "event.outcome": "failure", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "RcaSt872RG-R_WJBEGfYXA", "input.type": "log", "log.offset": 3978, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T22:49:34,859+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"transport\", \"event.action\":\"run_as_denied\", \"user.name\":\"user1\", \"user.run_as.name\":\"user1\", \"user.realm\":\"default_native\", \"user.run_as.realm\":\"default_native\", \"user.roles\":[\"test_role\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:52662\", \"request.id\":\"RcaSt872RG-R_WJBEGfYXA\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"alias1\"]}", @@ -395,6 +402,7 @@ "event.outcome": "success", "fileset.name": "audit", "host.id": "0RMNyghkQYCc_gVd1G6tZQ", + "http.request.id": "dGqPTdEQSX2TAPS3cvc1qA", "input.type": "log", "log.offset": 4463, "message": "{\"type\":\"audit\", \"timestamp\":\"2020-12-30T22:44:42,068+0200\", \"node.id\":\"0RMNyghkQYCc_gVd1G6tZQ\", \"event.type\":\"transport\", \"event.action\":\"run_as_granted\", \"user.name\":\"elastic\", \"user.run_as.name\":\"user1\", \"user.realm\":\"reserved\", \"user.run_as.realm\":\"default_native\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:52623\", \"request.id\":\"dGqPTdEQSX2TAPS3cvc1qA\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"alias1\"]}", diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 0eca28c60843..64fd7567ba1e 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,266 +1,272 @@ -description: Pipeline for parsing Nginx ingress controller access logs. Requires the +description: + Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' -- grok: - field: message - patterns: - - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" - %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} - "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} - %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] - \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_address_list}|-) - (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-) - (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} - pattern_definitions: - NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? - NGINX_NOTSEPARATOR: "[^\t ,:]+" - NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* - UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))* - UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* - UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* - UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* - ignore_missing: true -- grok: - field: nginx.ingress_controller.info - patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' - - "" - ignore_missing: true -- remove: - field: nginx.ingress_controller.info -- split: - field: nginx.ingress_controller.remote_ip_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.upstream_address_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.upstream.response.length_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.upstream.response.time_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.upstream.response.status_code_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.origin - separator: '"?,?\s+' - ignore_missing: true -- set: - field: source.address - if: ctx.source?.address == null - value: "" -- script: - if: ctx.nginx?.ingress_controller?.upstream?.response?.length_list != null && ctx.nginx.ingress_controller.upstream.response.length_list.length > 0 - lang: painless - source: >- - try { - if (ctx.nginx.ingress_controller.upstream.response.length_list.length == null) { - return; - } - int last_length = 0; - for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) { - last_length = Integer.parseInt(item); - } - ctx.nginx.ingress_controller.upstream.response.length = last_length; - } - catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.length = null; - } -- script: - if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0 - lang: painless - source: >- - try { - if (ctx.nginx.ingress_controller.upstream.response.time_list.length == null) { - return; - } - float res_time = 0; - for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { - res_time = res_time + Float.parseFloat(item); + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + - grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} + %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] + \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS_LIST:nginx.ingress_controller.upstream_address_list}|-) + (%{UPSTREAM_RESPONSE_LENGTH_LIST:nginx.ingress_controller.upstream.response.length_list}|-) (%{UPSTREAM_RESPONSE_TIME_LIST:nginx.ingress_controller.upstream.response.time_list}|-) + (%{UPSTREAM_RESPONSE_STATUS_CODE_LIST:nginx.ingress_controller.upstream.response.status_code_list}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))* + UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + ignore_missing: true + - grok: + field: nginx.ingress_controller.info + patterns: + - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}" + - "" + ignore_missing: true + - remove: + field: nginx.ingress_controller.info + - split: + field: nginx.ingress_controller.remote_ip_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.ingress_controller.upstream_address_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.ingress_controller.upstream.response.length_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.ingress_controller.upstream.response.time_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.ingress_controller.upstream.response.status_code_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.ingress_controller.origin + separator: '"?,?\s+' + ignore_missing: true + - set: + field: source.address + if: ctx.source?.address == null + value: "" + - set: + field: http.request.id + copy_from: nginx.ingress_controller.http.request.id + ignore_empty_value: true + ignore_failure: true + - script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.length_list != null && ctx.nginx.ingress_controller.upstream.response.length_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.length_list.length == null) { + return; + } + int last_length = 0; + for (def item : ctx.nginx.ingress_controller.upstream.response.length_list) { + last_length = Integer.parseInt(item); + } + ctx.nginx.ingress_controller.upstream.response.length = last_length; } - ctx.nginx.ingress_controller.upstream.response.time = res_time; - } - catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.time = null; - } -- script: - if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0 - lang: painless - source: >- - try { - if (ctx.nginx.ingress_controller.upstream.response.status_code_list.length == null) { - return; + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.length = null; } - int last_status_code; - for (def item : ctx.nginx.ingress_controller.upstream.response.status_code_list) { - last_status_code = Integer.parseInt(item); + - script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.time_list != null && ctx.nginx.ingress_controller.upstream.response.time_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.time_list.length == null) { + return; + } + float res_time = 0; + for (def item : ctx.nginx.ingress_controller.upstream.response.time_list) { + res_time = res_time + Float.parseFloat(item); + } + ctx.nginx.ingress_controller.upstream.response.time = res_time; } - ctx.nginx.ingress_controller.upstream.response.status_code = last_status_code; - } - catch (Exception e) { - ctx.nginx.ingress_controller.upstream.response.time = null; - } -- script: - if: ctx.nginx?.ingress_controller?.upstream_address_list != null && ctx.nginx.ingress_controller.upstream_address_list.length > 0 - lang: painless - source: >- - try { - if (ctx.nginx.ingress_controller.upstream_address_list.length == null) { - return; + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.time = null; } - def last_upstream = ""; - for (def item : ctx.nginx.ingress_controller.upstream_address_list) { - last_upstream = item; + - script: + if: ctx.nginx?.ingress_controller?.upstream?.response?.status_code_list != null && ctx.nginx.ingress_controller.upstream.response.status_code_list.length > 0 + lang: painless + source: >- + try { + if (ctx.nginx.ingress_controller.upstream.response.status_code_list.length == null) { + return; + } + int last_status_code; + for (def item : ctx.nginx.ingress_controller.upstream.response.status_code_list) { + last_status_code = Integer.parseInt(item); + } + ctx.nginx.ingress_controller.upstream.response.status_code = last_status_code; } - StringTokenizer tok = new StringTokenizer(last_upstream, ":"); - if (tok.countTokens()>1) { - ctx.nginx.ingress_controller.upstream.ip = tok.nextToken(); - ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken()); - } else { - ctx.nginx.ingress_controller.upstream.ip = last_upstream; + catch (Exception e) { + ctx.nginx.ingress_controller.upstream.response.time = null; } - } - catch (Exception e) { - ctx.nginx.ingress_controller.upstream.ip = null; - ctx.nginx.ingress_controller.upstream.port = null; - } -- script: - if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 - lang: painless - source: >- - boolean isPrivate(def dot, def ip) { + - script: + if: ctx.nginx?.ingress_controller?.upstream_address_list != null && ctx.nginx.ingress_controller.upstream_address_list.length > 0 + lang: painless + source: >- try { - StringTokenizer tok = new StringTokenizer(ip, dot); - int firstByte = Integer.parseInt(tok.nextToken()); - int secondByte = Integer.parseInt(tok.nextToken()); - if (firstByte == 10) { - return true; - } - if (firstByte == 192 && secondByte == 168) { - return true; + if (ctx.nginx.ingress_controller.upstream_address_list.length == null) { + return; } - if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { - return true; + def last_upstream = ""; + for (def item : ctx.nginx.ingress_controller.upstream_address_list) { + last_upstream = item; } - if (firstByte == 127) { - return true; + StringTokenizer tok = new StringTokenizer(last_upstream, ":"); + if (tok.countTokens()>1) { + ctx.nginx.ingress_controller.upstream.ip = tok.nextToken(); + ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken()); + } else { + ctx.nginx.ingress_controller.upstream.ip = last_upstream; } - return false; } catch (Exception e) { - return false; + ctx.nginx.ingress_controller.upstream.ip = null; + ctx.nginx.ingress_controller.upstream.port = null; } - } - try { - ctx.source.address = null; - if (ctx.nginx.ingress_controller.remote_ip_list == null) { - return; + - script: + if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 + lang: painless + source: >- + boolean isPrivate(def dot, def ip) { + try { + StringTokenizer tok = new StringTokenizer(ip, dot); + int firstByte = Integer.parseInt(tok.nextToken()); + int secondByte = Integer.parseInt(tok.nextToken()); + if (firstByte == 10) { + return true; + } + if (firstByte == 192 && secondByte == 168) { + return true; + } + if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { + return true; + } + if (firstByte == 127) { + return true; + } + return false; + } + catch (Exception e) { + return false; + } } - def found = false; - for (def item : ctx.nginx.ingress_controller.remote_ip_list) { - if (!isPrivate(params.dot, item)) { - ctx.source.address = item; - found = true; - break; + try { + ctx.source.address = null; + if (ctx.nginx.ingress_controller.remote_ip_list == null) { + return; + } + def found = false; + for (def item : ctx.nginx.ingress_controller.remote_ip_list) { + if (!isPrivate(params.dot, item)) { + ctx.source.address = item; + found = true; + break; + } + } + if (!found) { + ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; } } - if (!found) { - ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; + catch (Exception e) { + ctx.source.address = null; } - } - catch (Exception e) { - ctx.source.address = null; - } - params: - dot: . -- remove: - field: source.address - if: ctx.source.address == null -- grok: - field: source.address - patterns: - - ^%{IP:source.ip}$ - ignore_failure: true -- remove: - field: message -- rename: - field: '@timestamp' - target_field: event.created -- date: - field: nginx.ingress_controller.time - target_field: '@timestamp' - formats: - - dd/MMM/yyyy:H:m:s Z - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- remove: - field: nginx.ingress_controller.time -- user_agent: - field: user_agent.original - ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- set: - field: event.kind - value: event -- append: - field: event.category - value: web -- append: - field: event.type - value: info -- set: - field: event.outcome - value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" -- set: - field: event.outcome - value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx?.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx?.destination?.ip != null" -- append: - field: related.user - value: "{{user.name}}" - if: "ctx?.user?.name != null" + params: + dot: . + - remove: + field: source.address + if: ctx.source.address == null + - grok: + field: source.address + patterns: + - ^%{IP:source.ip}$ + ignore_failure: true + - remove: + field: message + - rename: + field: "@timestamp" + target_field: event.created + - date: + field: nginx.ingress_controller.time + target_field: "@timestamp" + formats: + - dd/MMM/yyyy:H:m:s Z + on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" + - remove: + field: nginx.ingress_controller.time + - user_agent: + field: user_agent.original + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - set: + field: event.kind + value: event + - append: + field: event.category + value: web + - append: + field: event.type + value: info + - set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + - set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + - append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" + - append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index e55c45814deb..02e205d81c39 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -13,6 +13,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "529a007902362a5f51385a5fa7049884", "http.request.method": "POST", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -71,6 +72,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "68fa971ce4dfce685fdc01c877bfa645", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -129,6 +131,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "0be411044cb1cb67580e115413b2da60", "http.request.method": "DELETE", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -187,6 +190,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "f479ab1d9cc8afcbac9e9f958ff8babc", "http.request.method": "PATCH", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -245,6 +249,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "4c7d2079340e68353c7d0dfff00b904b", "http.request.method": "PATCHp", "http.response.body.bytes": 163, "http.response.status_code": 400, @@ -281,6 +286,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "efb0c5aa8be6cdeb4a7e7bd090e3d893", "http.request.method": "geti", "http.response.body.bytes": 163, "http.response.status_code": 400, @@ -317,6 +323,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "457b71c3e1ee1887bb809effd301a0ec", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -375,6 +382,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "da29abf31e4d6324cebe5e7bca370709", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -436,6 +444,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "e983c8cf3d713548baa50c9e2fffeb34", "http.request.method": "GET", "http.request.referrer": "http://hello-world.info/products/42", "http.response.body.bytes": 59, @@ -498,6 +507,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "3d7ff18ff4181a7db5013a76f975d900", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, @@ -559,6 +569,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "d131fe4bcd359cf947f75efca4bfa553", "http.request.method": "GET", "http.request.referrer": "http://hello-world.info/v2", "http.response.body.bytes": 59, @@ -621,6 +632,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "ef6629fcaaa1ea0d1a843cf2bf40571d", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -682,6 +694,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "2593a1126588922449c183c8d9ddbbeb", "http.request.method": "GET", "http.request.referrer": "http://hello-world.info/products/42", "http.response.body.bytes": 59, @@ -744,6 +757,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "0f76ea730f282d5759018eb756b23b14", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -805,6 +819,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "21efd18e3a7952fc78c0f2dcc1f05e69", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -866,6 +881,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "e096809c2cb46f004c3b538b23916e5b", "http.request.method": "GET", "http.request.referrer": "http://hello-world.info/", "http.response.body.bytes": 59, @@ -928,6 +944,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "0a2a92d080e664dd4e95c85d097c9d3d", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, @@ -989,6 +1006,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "4c024ebfc20acfb2d59e542e3ed60789", "http.request.method": "GET", "http.request.referrer": "http://hello-world.info/v2", "http.response.body.bytes": 59, @@ -1051,6 +1069,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "9a7babf34ca4ee59d90ac48d452a9214", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -1109,6 +1128,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "ba91c30454893c121879396b0a78be79", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, @@ -1170,6 +1190,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "98c81aa2d50c67f6fb1fa16d5ce62f8f", "http.request.method": "GET", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -1231,6 +1252,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, @@ -1292,6 +1314,7 @@ "info" ], "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", "http.request.method": "GET", "http.response.body.bytes": 61, "http.response.status_code": 200, From 9f01bc729e97f2a8ceb7ee10ca211c6397fbbec1 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Thu, 8 Apr 2021 22:54:10 -0400 Subject: [PATCH 09/12] Fix inode removal tracking code (#25002) * Fix inode removal tracking code * update changelog --- CHANGELOG.next.asciidoc | 1 + libbeat/common/file/file_other.go | 9 +++++++-- libbeat/common/file/file_other_test.go | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e65221b62947..0d6ed4bade49 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -240,6 +240,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318] - Allow cgroup self-monitoring to see alternate `hostfs` paths {pull}24334[24334] - Fix 'make setup' instructions for a new beat {pull}24944[24944] +- Fix inode removal tracking code when files are replaced by files with the same name {pull}25002[25002] *Auditbeat* diff --git a/libbeat/common/file/file_other.go b/libbeat/common/file/file_other.go index fa2082da8ac1..86aa193f21b9 100644 --- a/libbeat/common/file/file_other.go +++ b/libbeat/common/file/file_other.go @@ -65,8 +65,13 @@ func ReadOpen(path string) (*os.File, error) { // IsRemoved checks wheter the file held by f is removed. func IsRemoved(f *os.File) bool { - _, err := os.Stat(f.Name()) - return err != nil + stat, err := f.Stat() + if err != nil { + // if we got an error from a Stat call just assume we are removed + return true + } + sysStat := stat.Sys().(*syscall.Stat_t) + return sysStat.Nlink == 0 } // InodeString returns the inode in string. diff --git a/libbeat/common/file/file_other_test.go b/libbeat/common/file/file_other_test.go index 79ebadbfc1e7..efc60b340db4 100644 --- a/libbeat/common/file/file_other_test.go +++ b/libbeat/common/file/file_other_test.go @@ -67,6 +67,20 @@ func TestGetOSFileStateStat(t *testing.T) { } } +func TestRemoved(t *testing.T) { + file, err := ioutil.TempFile("", "") + assert.NoError(t, err) + + assert.NoError(t, os.Remove(file.Name())) + + replaced, err := os.Create(file.Name()) + assert.NoError(t, err) + defer os.Remove(replaced.Name()) + defer replaced.Close() + + assert.True(t, IsRemoved(file)) +} + func BenchmarkStateString(b *testing.B) { var samples [50]uint64 for i, v := 0, uint64(0); i < len(samples); i, v = i+1, v+math.MaxUint64/uint64(len(samples)) { From 9625db682ee8ec28e3831c78b49c49f23cb2eeeb Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 9 Apr 2021 08:35:40 +0200 Subject: [PATCH 10/12] [Ingest Manager] Expose processes and their metrics (#24788) [Ingest Manager] Expose processes and their metrics (#24788) --- NOTICE.txt | 74 +++---- go.mod | 2 +- .../_meta/config/common.p2.yml.tmpl | 9 + .../_meta/config/common.reference.p2.yml.tmpl | 9 + .../config/elastic-agent.docker.yml.tmpl | 9 + x-pack/elastic-agent/_meta/elastic-agent.yml | 9 + x-pack/elastic-agent/elastic-agent.docker.yml | 9 + .../elastic-agent/elastic-agent.reference.yml | 9 + x-pack/elastic-agent/elastic-agent.yml | 9 + .../pkg/agent/application/application.go | 2 + .../application/fleet_server_bootstrap.go | 6 + .../pkg/agent/application/local_mode.go | 6 + .../pkg/agent/application/managed_mode.go | 6 + .../agent/application/pipeline/pipeline.go | 2 + .../application/pipeline/router/router.go | 4 + .../pipeline/stream/operator_stream.go | 13 ++ x-pack/elastic-agent/pkg/agent/cmd/inspect.go | 5 + x-pack/elastic-agent/pkg/agent/cmd/run.go | 76 +------ .../pkg/agent/operation/monitoring.go | 2 +- .../pkg/agent/operation/monitoring_test.go | 5 +- .../core/monitoring/beats/beats_monitor.go | 2 +- .../pkg/core/monitoring/beats/monitoring.go | 16 +- .../pkg/core/monitoring/config/config.go | 22 +- .../pkg/core/monitoring/server/handler.go | 47 +++++ .../pkg/core/monitoring/server/process.go | 195 ++++++++++++++++++ .../core/monitoring/server/process_test.go | 56 +++++ .../pkg/core/monitoring/server/processes.go | 133 ++++++++++++ .../core/monitoring/server/processes_test.go | 171 +++++++++++++++ .../pkg/core/monitoring/server/server.go | 109 ++++++++++ .../pkg/core/monitoring/server/stats.go | 36 ++++ 30 files changed, 936 insertions(+), 117 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/handler.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/process.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/process_test.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/processes.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/processes_test.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/server.go create mode 100644 x-pack/elastic-agent/pkg/core/monitoring/server/stats.go diff --git a/NOTICE.txt b/NOTICE.txt index db86aec50f91..96cd4f823748 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10242,6 +10242,43 @@ Contents of probable licence file $GOMODCACHE/github.com/gorhill/cronexpr@v0.0.0 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/gorilla/mux +Version: v1.7.2 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gorilla/mux@v1.7.2/LICENSE: + +Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/h2non/filetype Version: v1.1.1-0.20201130172452-f60988ab73d5 @@ -31247,43 +31284,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/gorilla/mux -Version: v1.7.2 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/gorilla/mux@v1.7.2/LICENSE: - -Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/gorilla/websocket Version: v1.4.1 diff --git a/go.mod b/go.mod index 15bb085efab9..3763a3577985 100644 --- a/go.mod +++ b/go.mod @@ -95,7 +95,7 @@ require ( github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 github.com/google/uuid v1.1.2 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 - github.com/gorilla/mux v1.7.2 // indirect + github.com/gorilla/mux v1.7.2 github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect github.com/h2non/filetype v1.1.1-0.20201130172452-f60988ab73d5 github.com/hashicorp/go-multierror v1.1.0 diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index d73ad623a26a..b706c5ba8019 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -33,6 +33,15 @@ inputs: # logs: true # # enables metrics monitoring # metrics: true +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index 055f6444f166..07ff537ee7fd 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -107,6 +107,15 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 1e4c71788473..89709c810c66 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -107,6 +107,15 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml index 50d435c4a5cf..09533381ff63 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.yml @@ -102,6 +102,15 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload his configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index f20bb9a0ad97..c65c2b07219d 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -107,6 +107,15 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 28c9dc8b83dd..023e3ef8aeef 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -113,6 +113,15 @@ inputs: # logs: false # # enables metrics monitoring # metrics: false +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index cd19a339b3c7..db9568fded4d 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -39,6 +39,15 @@ inputs: # logs: true # # enables metrics monitoring # metrics: true +# # exposes agent metrics using http, by default sockets and named pipes are used +# http: +# # enables http endpoint +# enabled: false +# # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# # When using IP addresses, it is recommended to only use localhost. +# host: localhost +# # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled. +# port: 0 # # Allow fleet to reload its configuration locally on disk. # # Notes: Only specific process configuration will be reloaded. diff --git a/x-pack/elastic-agent/pkg/agent/application/application.go b/x-pack/elastic-agent/pkg/agent/application/application.go index 96dcac99dff5..53c74afabe50 100644 --- a/x-pack/elastic-agent/pkg/agent/application/application.go +++ b/x-pack/elastic-agent/pkg/agent/application/application.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/status" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" @@ -26,6 +27,7 @@ type Application interface { Start() error Stop() error AgentInfo() *info.AgentInfo + Routes() *sorted.Set } type reexecManager interface { diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_server_bootstrap.go b/x-pack/elastic-agent/pkg/agent/application/fleet_server_bootstrap.go index 6f3dd09335b0..60be85147cfe 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_server_bootstrap.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_server_bootstrap.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" "github.com/elastic/go-sysinfo" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" @@ -113,6 +114,11 @@ func newFleetServerBootstrap( return bootstrapApp, nil } +// Routes returns a list of routes handled by server. +func (b *FleetServerBootstrap) Routes() *sorted.Set { + return b.router.Routes() +} + // Start starts a managed elastic-agent. func (b *FleetServerBootstrap) Start() error { b.log.Info("Agent is starting") diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go index ffb59f281ff6..4da6304e1ccf 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go @@ -30,6 +30,7 @@ import ( acker "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/acker/noop" reporting "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter" logreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/log" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" ) type discoverFunc func() ([]string, error) @@ -157,6 +158,11 @@ func newLocal( return localApplication, nil } +// Routes returns a list of routes handled by agent. +func (l *Local) Routes() *sorted.Set { + return l.router.Routes() +} + // Start starts a local agent. func (l *Local) Start() error { l.log.Info("Agent is starting") diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 94a8a826de49..e85034de4d43 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -43,6 +43,7 @@ import ( reporting "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter" fleetreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/fleet" logreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/log" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" ) type stateStore interface { @@ -278,6 +279,11 @@ func newManaged( return managedApplication, nil } +// Routes returns a list of routes handled by agent. +func (m *Managed) Routes() *sorted.Set { + return m.router.Routes() +} + // Start starts a managed elastic-agent. func (m *Managed) Start() error { m.log.Info("Agent is starting") diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go index 8286c1ee3a45..c8cac5b32162 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" ) // ConfigHandler is capable of handling configrequest. @@ -30,6 +31,7 @@ type RoutingKey = string // Router is an interace routes programs to correspongind stream type Router interface { + Routes() *sorted.Set Route(id string, grpProg map[RoutingKey][]program.Program) error Shutdown() } diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/router/router.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/router/router.go index 6c7a27a2bd99..bda4a7b7cde7 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/router/router.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/router/router.go @@ -34,6 +34,10 @@ func New(log *logger.Logger, factory pipeline.StreamFunc) (pipeline.Router, erro return &router{log: log, streamFactory: factory, routes: sorted.NewSet()}, nil } +func (r *router) Routes() *sorted.Set { + return r.routes +} + func (r *router) Route(id string, grpProg map[pipeline.RoutingKey][]program.Program) error { s := sorted.NewSet() diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/stream/operator_stream.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/stream/operator_stream.go index 519a7b6bb523..57b16bbabc4e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/stream/operator_stream.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/stream/operator_stream.go @@ -8,6 +8,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" ) type operatorStream struct { @@ -15,10 +16,22 @@ type operatorStream struct { log *logger.Logger } +type stater interface { + State() map[string]state.State +} + func (b *operatorStream) Close() error { return b.configHandler.Close() } +func (b *operatorStream) State() map[string]state.State { + if s, ok := b.configHandler.(stater); ok { + return s.State() + } + + return nil +} + func (b *operatorStream) Execute(cfg configrequest.Request) error { return b.configHandler.HandleConfig(cfg) } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/inspect.go b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go index f13163f27b29..294a8bda284f 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/inspect.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go @@ -31,6 +31,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/noop" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/status" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" "github.com/elastic/go-sysinfo" ) @@ -299,6 +300,10 @@ type inmemRouter struct { programs map[string][]program.Program } +func (r *inmemRouter) Routes() *sorted.Set { + return nil +} + func (r *inmemRouter) Route(id string, grpProg map[pipeline.RoutingKey][]program.Program) error { r.programs = grpProg return nil diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 75d9e49fad85..2c0bb76cdb0b 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -6,14 +6,10 @@ package cmd import ( "context" - "encoding/json" "fmt" - "net/http" "os" "os/signal" "path/filepath" - "runtime" - "strings" "syscall" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/status" @@ -22,7 +18,6 @@ import ( "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/cmd/instance/metrics" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" @@ -39,6 +34,8 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/beats" + monitoringCfg "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/config" + monitoringServer "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -157,7 +154,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { // Windows: Mark return err } - serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS()) + serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app) if err != nil { return err } @@ -265,7 +262,7 @@ func defaultLogLevel(cfg *configuration.Configuration) string { return defaultLogLevel } -func setupMetrics(agentInfo *info.AgentInfo, logger *logger.Logger, operatingSystem string) (func() error, error) { +func setupMetrics(agentInfo *info.AgentInfo, logger *logger.Logger, operatingSystem string, cfg *monitoringCfg.MonitoringConfig, app application.Application) (func() error, error) { // use libbeat to setup metrics if err := metrics.SetupMetrics(agentName); err != nil { return nil, err @@ -274,18 +271,10 @@ func setupMetrics(agentInfo *info.AgentInfo, logger *logger.Logger, operatingSys // start server for stats endpointConfig := api.Config{ Enabled: true, - Host: beats.AgentMonitoringEndpoint(operatingSystem), + Host: beats.AgentMonitoringEndpoint(operatingSystem, cfg.HTTP), } - // create agent config path - createAgentMonitoringDrop(endpointConfig.Host) - - cfg, err := common.NewConfigFrom(endpointConfig) - if err != nil { - return nil, err - } - - s, err := exposeMetricsEndpoint(logger, cfg, monitoring.GetNamespace) + s, err := monitoringServer.New(logger, endpointConfig, monitoring.GetNamespace, app.Routes, isProcessStatsEnabled(cfg.HTTP)) if err != nil { return nil, errors.New(err, "could not start the HTTP server for the API") } @@ -295,55 +284,6 @@ func setupMetrics(agentInfo *info.AgentInfo, logger *logger.Logger, operatingSys return s.Stop, nil } -func createAgentMonitoringDrop(drop string) error { - if drop == "" || runtime.GOOS == "windows" { - return nil - } - - path := strings.TrimPrefix(drop, "unix://") - if strings.HasSuffix(path, ".sock") { - path = filepath.Dir(path) - } - - _, err := os.Stat(path) - if err != nil { - if !os.IsNotExist(err) { - return err - } - - // create - if err := os.MkdirAll(path, 0775); err != nil { - return err - } - } - - return os.Chown(path, os.Geteuid(), os.Getegid()) -} - -func exposeMetricsEndpoint(log *logger.Logger, config *common.Config, ns func(string) *monitoring.Namespace) (*api.Server, error) { - mux := http.NewServeMux() - - makeAPIHandler := func(ns *monitoring.Namespace) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - - data := monitoring.CollectStructSnapshot( - ns.GetRegistry(), - monitoring.Full, - false, - ) - - bytes, err := json.Marshal(data) - var content string - if err != nil { - content = fmt.Sprintf("Not valid json: %v", err) - } else { - content = string(bytes) - } - fmt.Fprint(w, content) - } - } - - mux.HandleFunc("/stats", makeAPIHandler(ns("stats"))) - return api.New(log, mux, config) +func isProcessStatsEnabled(cfg *monitoringCfg.MonitoringHTTPConfig) bool { + return cfg != nil && cfg.Enabled } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index f5a5f54545db..4fa440df5371 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -439,7 +439,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "namespace": "agent", "period": "10s", "path": "/stats", - "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS())}, + "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)}, "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), "processors": []map[string]interface{}{ { diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 8a25eb809351..cbf9edf3266c 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -50,7 +50,7 @@ func TestGenerateSteps(t *testing.T) { for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { m := &testMonitor{monitorLogs: tc.Config.MonitorLogs, monitorMetrics: tc.Config.MonitorMetrics} - operator := getMonitorableTestOperator(t, "tests/scripts", m) + operator := getMonitorableTestOperator(t, "tests/scripts", m, tc.Config) steps := operator.generateMonitoringSteps("8.0", sampleOutput) if actualSteps := len(steps); actualSteps != tc.ExpectedSteps { t.Fatalf("invalid number of steps, expected %v, got %v", tc.ExpectedSteps, actualSteps) @@ -100,7 +100,7 @@ func checkStep(t *testing.T, stepName string, expectedOutput interface{}, s conf } } -func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.Monitor) *Operator { +func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.Monitor, mcfg *monitoringConfig.MonitoringConfig) *Operator { cfg := &configuration.SettingsConfig{ RetryConfig: &retry.Config{ Enabled: true, @@ -113,6 +113,7 @@ func getMonitorableTestOperator(t *testing.T, installPath string, m monitoring.M InstallPath: installPath, OperatingSystem: "darwin", }, + MonitoringConfig: mcfg, } l := getLogger() diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index f2066bf04ec6..743d44118d6b 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -76,7 +76,7 @@ func (b *Monitor) WatchLogs() bool { return b.config.Enabled && b.config.Monitor func (b *Monitor) WatchMetrics() bool { return b.config.Enabled && b.config.MonitorMetrics } func (b *Monitor) generateMonitoringEndpoint(spec program.Spec, pipelineID string) string { - return getMonitoringEndpoint(spec, b.operatingSystem, pipelineID) + return MonitoringEndpoint(spec, b.operatingSystem, pipelineID) } func (b *Monitor) generateLoggingFile(spec program.Spec, pipelineID string) string { diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go index b10f0ef82a59..6aa6c471e791 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" + monitoringConfig "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/config" ) const ( @@ -24,9 +25,12 @@ const ( // args: pipeline name, application name agentMbEndpointFileFormatWin = `npipe:///elastic-agent` + // agentMbEndpointHTTP is used with cloud and exposes metrics on http endpoint + agentMbEndpointHTTP = "http://localhost:%d" ) -func getMonitoringEndpoint(spec program.Spec, operatingSystem, pipelineID string) string { +// MonitoringEndpoint is an endpoint where process is exposing its metrics. +func MonitoringEndpoint(spec program.Spec, operatingSystem, pipelineID string) string { if endpoint, ok := spec.MetricEndpoints[operatingSystem]; ok { return endpoint } @@ -54,7 +58,11 @@ func getLoggingFile(spec program.Spec, operatingSystem, installPath, pipelineID } // AgentMonitoringEndpoint returns endpoint with exposed metrics for agent. -func AgentMonitoringEndpoint(operatingSystem string) string { +func AgentMonitoringEndpoint(operatingSystem string, cfg *monitoringConfig.MonitoringHTTPConfig) string { + if cfg != nil && cfg.Enabled { + return fmt.Sprintf(agentMbEndpointHTTP, cfg.Port) + } + if operatingSystem == "windows" { return agentMbEndpointFileFormatWin } @@ -69,6 +77,6 @@ func AgentMonitoringEndpoint(operatingSystem string) string { } // AgentPrefixedMonitoringEndpoint returns endpoint with exposed metrics for agent. -func AgentPrefixedMonitoringEndpoint(operatingSystem string) string { - return httpPlusPrefix + AgentMonitoringEndpoint(operatingSystem) +func AgentPrefixedMonitoringEndpoint(operatingSystem string, cfg *monitoringConfig.MonitoringHTTPConfig) string { + return httpPlusPrefix + AgentMonitoringEndpoint(operatingSystem, cfg) } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index ceb4b3b6a56e..2ce067d4e192 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -4,11 +4,23 @@ package config +const defaultPort = 6791 + // MonitoringConfig describes a configuration of a monitoring type MonitoringConfig struct { - Enabled bool `yaml:"enabled" config:"enabled"` - MonitorLogs bool `yaml:"logs" config:"logs"` - MonitorMetrics bool `yaml:"metrics" config:"metrics"` + Enabled bool `yaml:"enabled" config:"enabled"` + MonitorLogs bool `yaml:"logs" config:"logs"` + MonitorMetrics bool `yaml:"metrics" config:"metrics"` + HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` +} + +// MonitoringHTTPConfig is a config defining HTTP endpoint published by agent +// for other processes to watch its metrics. +// Processes are only exposed when HTTP is enabled. +type MonitoringHTTPConfig struct { + Enabled bool `yaml:"enabled" config:"enabled"` + Host string `yaml:"host" config:"host"` + Port int `yaml:"port" config:"port" validate:"min=0,max=65535,nonzero"` } // DefaultConfig creates a config with pre-set default values. @@ -17,5 +29,9 @@ func DefaultConfig() *MonitoringConfig { Enabled: true, MonitorLogs: true, MonitorMetrics: true, + HTTP: &MonitoringHTTPConfig{ + Enabled: false, + Port: defaultPort, + }, } } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/handler.go b/x-pack/elastic-agent/pkg/core/monitoring/server/handler.go new file mode 100644 index 000000000000..dfb50d7e0246 --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/handler.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "fmt" + "net/http" +) + +type apiError interface { + Status() int +} + +func createHandler(fn func(w http.ResponseWriter, r *http.Request) error) *apiHandler { + return &apiHandler{ + innerFn: fn, + } +} + +type apiHandler struct { + innerFn func(w http.ResponseWriter, r *http.Request) error +} + +// ServeHTTP sets status code based on err returned +func (h *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + err := h.innerFn(w, r) + if err != nil { + writeResponse(w, unexpectedErrorWithReason(err.Error())) + + switch e := err.(type) { + case apiError: + w.WriteHeader(e.Status()) + default: + w.WriteHeader(http.StatusInternalServerError) + + } + } +} + +func unexpectedErrorWithReason(reason string, args ...interface{}) errResponse { + return errResponse{ + Type: errTypeUnexpected, + Reason: fmt.Sprintf(reason, args...), + } +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/process.go b/x-pack/elastic-agent/pkg/core/monitoring/server/process.go new file mode 100644 index 000000000000..753fe3602b59 --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/process.go @@ -0,0 +1,195 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + "syscall" + "time" + + "github.com/gorilla/mux" + + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/beats" +) + +const ( + processIDKey = "processID" + monitoringSuffix = "-monitoring" + separator = "-" + timeout = 10 * time.Second + errTypeUnexpected = "UNEXPECTED" + + httpPlusPrefix = "http+" +) + +var ( + // ErrProgramNotSupported returned when requesting metrics for not supported program. + ErrProgramNotSupported = errors.New("specified program is not supported") + invalidChars = map[rune]struct{}{ + '"': {}, + '<': {}, + '>': {}, + '|': {}, + 0: {}, + ':': {}, + '*': {}, + '?': {}, + '\\': {}, + '/': {}, + ';': {}, + } +) + +func processHandler() func(http.ResponseWriter, *http.Request) error { + return func(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + vars := mux.Vars(r) + id, found := vars[processIDKey] + + if !found { + return errorfWithStatus(http.StatusNotFound, "productID not found") + } + + metricsBytes, statusCode, metricsErr := processMetrics(r.Context(), id) + if metricsErr != nil { + return metricsErr + } + + if statusCode > 0 { + w.WriteHeader(statusCode) + } + + fmt.Fprint(w, string(metricsBytes)) + return nil + } +} + +func processMetrics(ctx context.Context, id string) ([]byte, int, error) { + detail, err := parseID(id) + if err != nil { + return nil, 0, err + } + + endpoint := beats.MonitoringEndpoint(detail.spec, artifact.DefaultConfig().OS(), detail.output) + if !strings.HasPrefix(endpoint, httpPlusPrefix) && !strings.HasPrefix(endpoint, "http") { + // add prefix for npipe and unix + endpoint = httpPlusPrefix + endpoint + } + + if detail.isMonitoring { + endpoint += "_monitor" + } + + hostData, err := parse.ParseURL(endpoint, "http", "", "", "stats", "") + if err != nil { + return nil, 0, errorWithStatus(http.StatusInternalServerError, err) + } + + dialer, err := hostData.Transport.Make(timeout) + if err != nil { + return nil, 0, errorWithStatus(http.StatusInternalServerError, err) + } + + client := http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + Dial: dialer.Dial, + }, + } + + req, err := http.NewRequest("GET", hostData.URI, nil) + if err != nil { + return nil, 0, errorWithStatus( + http.StatusInternalServerError, + fmt.Errorf("fetching metrics failed: %v", err.Error()), + ) + } + + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + statusCode := http.StatusInternalServerError + if errors.Is(err, syscall.ENOENT) { + statusCode = http.StatusNotFound + } + return nil, 0, errorWithStatus(statusCode, err) + } + defer resp.Body.Close() + + rb, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, 0, errorWithStatus(http.StatusInternalServerError, err) + } + + return rb, resp.StatusCode, nil +} + +func writeResponse(w http.ResponseWriter, c interface{}) { + bytes, err := json.Marshal(c) + if err != nil { + // json marshal failed + fmt.Fprintf(w, "Not valid json: %v", err) + return + } + + fmt.Fprint(w, string(bytes)) + +} + +type programDetail struct { + output string + binaryName string + isMonitoring bool + spec program.Spec +} + +func parseID(id string) (programDetail, error) { + var detail programDetail + if !isIDValid(id) { + return detail, errorfWithStatus(http.StatusBadRequest, "provided ID is not valid") + } + + for p, spec := range program.SupportedMap { + if !strings.HasPrefix(id, p+separator) { + continue + } + + detail.binaryName = p + detail.spec = spec + break + } + + if detail.binaryName == "" { + return detail, errorWithStatus(http.StatusNotFound, ErrProgramNotSupported) + } + + if strings.HasSuffix(id, monitoringSuffix) { + detail.isMonitoring = true + id = strings.TrimSuffix(id, monitoringSuffix) + } + + detail.output = strings.TrimPrefix(id, detail.binaryName+separator) + + return detail, nil +} + +func isIDValid(id string) bool { + for _, c := range id { + if _, found := invalidChars[c]; found { + return false + } + } + + return true +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/process_test.go b/x-pack/elastic-agent/pkg/core/monitoring/server/process_test.go new file mode 100644 index 000000000000..e518322749b8 --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/process_test.go @@ -0,0 +1,56 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package server + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseID(t *testing.T) { + cases := []struct { + Name string + ID string + ExpectedError bool + ExpectedStatusCode int + ExpectedProgram programDetail + }{ + {"path injected id", ".././../etc/passwd", true, http.StatusBadRequest, programDetail{}}, + {"pipe injected id", "first | second", true, http.StatusBadRequest, programDetail{}}, + {"filebeat with suffix", "filebeat;cat demo-default-monitoring", true, http.StatusBadRequest, programDetail{}}, + + {"filebeat correct", "filebeat-default", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "filebeat"}}, + {"filebeat monitor correct", "filebeat-default-monitoring", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "filebeat", isMonitoring: true}}, + + {"mb correct", "metricbeat-default", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "metricbeat"}}, + {"mb monitor correct", "metricbeat-default-monitoring", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "metricbeat", isMonitoring: true}}, + + {"endpoint correct", "endpoint-security-default", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "endpoint-security"}}, + {"endpoint monitor correct", "endpoint-security-default-monitoring", false, http.StatusBadRequest, programDetail{output: "default", binaryName: "endpoint-security", isMonitoring: true}}, + + {"unknown", "unknown-default", true, http.StatusNotFound, programDetail{}}, + {"unknown monitor", "unknown-default-monitoring", true, http.StatusNotFound, programDetail{}}, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + pd, err := parseID(tc.ID) + if !tc.ExpectedError { + require.NoError(t, err) + } + + if tc.ExpectedStatusCode > 0 && tc.ExpectedError { + statErr, ok := err.(apiError) + require.True(t, ok) + require.Equal(t, tc.ExpectedStatusCode, statErr.Status()) + } + + require.Equal(t, tc.ExpectedProgram.binaryName, pd.binaryName) + require.Equal(t, tc.ExpectedProgram.output, pd.output) + require.Equal(t, tc.ExpectedProgram.isMonitoring, pd.isMonitoring) + }) + } +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/processes.go b/x-pack/elastic-agent/pkg/core/monitoring/server/processes.go new file mode 100644 index 000000000000..a298761065c7 --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/processes.go @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "net/http" + "strconv" + "strings" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" +) + +const ( + configuredType = "configured" + internalType = "internal" +) + +type sourceInfo struct { + // Kind is a kind of process e.g configured or internal + // configured - used for user configured processes + // internal - used for monitoring processes + Kind string `json:"kind"` + + // Outputs process is handling. + Outputs []string `json:"outputs"` +} + +type processInfo struct { + // ID is a unique id of the process. + ID string `json:"id"` + + // PID is a current process ID. + PID string `json:"pid"` + + // Binary name e.g filebeat, this does not contain absolute path. + Binary string `json:"binary"` + + // Source information + Source sourceInfo `json:"source"` +} + +type processesResponse struct { + Processes []processInfo `json:"processes"` +} + +type errResponse struct { + // Type is a type of error + Type string `json:"type"` + + // Reason is a detailed error message + Reason string `json:"reason"` +} + +type stater interface { + State() map[string]state.State +} + +func processesHandler(routesFetchFn func() *sorted.Set) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + resp := processesResponse{ + Processes: processesFromRoutes(routesFetchFn), + } + + writeResponse(w, resp) + } +} + +func processesFromRoutes(routesFetchFn func() *sorted.Set) []processInfo { + var processes []processInfo + routes := routesFetchFn() + + for _, k := range routes.Keys() { + op, found := routes.Get(k) + if !found { + continue + } + + s, ok := op.(stater) + if !ok { + continue + } + + states := s.State() + + for app, state := range states { + binaryName, isMonitoring := appNameFromDescriptor(app) + appType := configuredType + if isMonitoring { + appType = internalType + } + + var pid int + if state.ProcessInfo != nil { + pid = state.ProcessInfo.PID + } + + processInfo := processInfo{ + ID: processID(k, binaryName, isMonitoring), + PID: strconv.Itoa(pid), + Binary: binaryName, + Source: sourceInfo{ + Kind: appType, + Outputs: []string{k}, + }, + } + + processes = append(processes, processInfo) + } + } + + return processes +} + +func processID(output, binaryName string, isMonitoring bool) string { + id := binaryName + separator + output + if isMonitoring { + return id + monitoringSuffix + } + + return id +} + +func appNameFromDescriptor(d string) (string, bool) { + // monitoring desctiptor contains suffix with tag + // non monitoring just `binaryname--version` + parts := strings.Split(d, "--") + return parts[0], len(parts) > 2 +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/processes_test.go b/x-pack/elastic-agent/pkg/core/monitoring/server/processes_test.go new file mode 100644 index 000000000000..d1039c716433 --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/processes_test.go @@ -0,0 +1,171 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" +) + +func TestProcesses(t *testing.T) { + testRoutes := func(routes map[string]stater) func() *sorted.Set { + set := sorted.NewSet() + for k, s := range routes { + set.Add(k, s) + } + + return func() *sorted.Set { return set } + } + + t.Run("nothing running", func(t *testing.T) { + r := testRoutes(nil) + w := &testWriter{} + fn := processesHandler(r) + fn(w, nil) + + pr := processesResponse{ + Processes: nil, + } + + assert.Equal(t, 1, len(w.responses)) + if !assert.True(t, jsonComparer(w.responses[0], pr)) { + diff := cmp.Diff(pr, w.responses[0]) + t.Logf("Mismatch (-want, +got)\n%s", diff) + } + }) + + t.Run("process running", func(t *testing.T) { + r := testRoutes(map[string]stater{ + "default": &testStater{ + states: map[string]state.State{ + "filebeat--8.0.0": { + ProcessInfo: &process.Info{ + PID: 123, + Process: &os.Process{ + Pid: 123, + }, + }, + Status: state.Configuring, + }, + }, + }, + }) + w := &testWriter{} + fn := processesHandler(r) + fn(w, nil) + + pr := processesResponse{ + Processes: []processInfo{ + { + ID: "filebeat-default", + PID: "123", + Binary: "filebeat", + Source: sourceInfo{Kind: "configured", Outputs: []string{"default"}}, + }, + }, + } + + assert.Equal(t, 1, len(w.responses)) + if !assert.True(t, jsonComparer(w.responses[0], pr)) { + diff := cmp.Diff(w.responses[0], pr) + t.Logf("Mismatch (-want, +got)\n%s", diff) + } + }) + + t.Run("monitoring running", func(t *testing.T) { + r := testRoutes(map[string]stater{ + "default": &testStater{ + states: map[string]state.State{ + "filebeat--8.0.0--tag": { + ProcessInfo: &process.Info{ + PID: 123, + Process: &os.Process{ + Pid: 123, + }, + }, + Status: state.Configuring, + }, + }, + }, + }) + w := &testWriter{} + fn := processesHandler(r) + fn(w, nil) + + pr := processesResponse{ + Processes: []processInfo{ + { + ID: "filebeat-default-monitoring", + PID: "123", + Binary: "filebeat", + Source: sourceInfo{Kind: "internal", Outputs: []string{"default"}}, + }, + }, + } + + assert.Equal(t, 1, len(w.responses)) + if !assert.True(t, jsonComparer(w.responses[0], pr)) { + diff := cmp.Diff(w.responses[0], pr) + t.Logf("Mismatch (-want, +got)\n%s", diff) + } + }) +} + +type testStater struct { + states map[string]state.State +} + +func (s *testStater) State() map[string]state.State { + return s.states +} + +type testWriter struct { + responses []string + statusCode int +} + +func (w *testWriter) Header() http.Header { + return http.Header{} +} + +func (w *testWriter) Write(r []byte) (int, error) { + if w.responses == nil { + w.responses = make([]string, 0) + } + w.responses = append(w.responses, string(r)) + + return len(r), nil +} + +func (w *testWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode +} + +func jsonComparer(expected string, candidate interface{}) bool { + candidateJSON, err := json.Marshal(&candidate) + if err != nil { + fmt.Println(err) + return false + } + + cbytes := make([]byte, 0, len(candidateJSON)) + bbuf := bytes.NewBuffer(cbytes) + if err := json.Compact(bbuf, candidateJSON); err != nil { + fmt.Println(err) + return false + } + + return bytes.Equal([]byte(expected), bbuf.Bytes()) +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/server.go b/x-pack/elastic-agent/pkg/core/monitoring/server/server.go new file mode 100644 index 000000000000..25703d1524df --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/server.go @@ -0,0 +1,109 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/gorilla/mux" + + "github.com/elastic/beats/v7/libbeat/api" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/monitoring" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted" +) + +// New creates a new server exposing metrics and process information. +func New( + log *logger.Logger, + endpointConfig api.Config, + ns func(string) *monitoring.Namespace, + routesFetchFn func() *sorted.Set, + enableProcessStats bool, +) (*api.Server, error) { + if err := createAgentMonitoringDrop(endpointConfig.Host); err != nil { + // log but ignore + log.Errorf("failed to create monitoring drop: %v", err) + } + + cfg, err := common.NewConfigFrom(endpointConfig) + if err != nil { + return nil, err + } + + return exposeMetricsEndpoint(log, cfg, ns, routesFetchFn, enableProcessStats) +} + +func exposeMetricsEndpoint(log *logger.Logger, config *common.Config, ns func(string) *monitoring.Namespace, routesFetchFn func() *sorted.Set, enableProcessStats bool) (*api.Server, error) { + r := mux.NewRouter() + r.Handle("/stats", createHandler(statsHandler(ns("stats")))) + + if enableProcessStats { + r.HandleFunc("/processes", processesHandler(routesFetchFn)) + r.Handle("/processes/{processID}", createHandler(processHandler())) + } + + mux := http.NewServeMux() + mux.Handle("/", r) + + return api.New(log, mux, config) +} + +func createAgentMonitoringDrop(drop string) error { + if drop == "" || runtime.GOOS == "windows" { + return nil + } + + path := strings.TrimPrefix(drop, "unix://") + if strings.HasSuffix(path, ".sock") { + path = filepath.Dir(path) + } + + _, err := os.Stat(path) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + // create + if err := os.MkdirAll(path, 0775); err != nil { + return err + } + } + + return os.Chown(path, os.Geteuid(), os.Getegid()) +} + +func errorWithStatus(status int, err error) *statusError { + return &statusError{ + err: err, + status: status, + } +} + +func errorfWithStatus(status int, msg string, args ...string) *statusError { + err := fmt.Errorf(msg, args) + return errorWithStatus(status, err) +} + +// StatusError holds correlation between error and a status +type statusError struct { + err error + status int +} + +func (s *statusError) Status() int { + return s.status +} + +func (s *statusError) Error() string { + return s.err.Error() +} diff --git a/x-pack/elastic-agent/pkg/core/monitoring/server/stats.go b/x-pack/elastic-agent/pkg/core/monitoring/server/stats.go new file mode 100644 index 000000000000..5bb84246e9fd --- /dev/null +++ b/x-pack/elastic-agent/pkg/core/monitoring/server/stats.go @@ -0,0 +1,36 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/elastic/beats/v7/libbeat/monitoring" +) + +func statsHandler(ns *monitoring.Namespace) func(http.ResponseWriter, *http.Request) error { + return func(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + data := monitoring.CollectStructSnapshot( + ns.GetRegistry(), + monitoring.Full, + false, + ) + + bytes, err := json.Marshal(data) + var content string + if err != nil { + content = fmt.Sprintf("Not valid json: %v", err) + } else { + content = string(bytes) + } + fmt.Fprint(w, content) + + return nil + } +} From 5bde9d08c53b6b90cc7334e687f863e08c9d5d5d Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 9 Apr 2021 10:27:39 -0400 Subject: [PATCH 11/12] Add cloud.service.name to add_cloud_metadata (#24993) * Add cloud.service.name to add_cloud_metadata * update changelog --- CHANGELOG.next.asciidoc | 1 + .../provider_alibaba_cloud.go | 3 +++ .../provider_alibaba_cloud_test.go | 3 +++ .../add_cloud_metadata/provider_aws_ec2.go | 8 ++++++-- .../add_cloud_metadata/provider_aws_ec2_test.go | 17 ++++++++++++++++- .../add_cloud_metadata/provider_azure_vm.go | 4 ++++ .../provider_azure_vm_test.go | 3 +++ .../provider_digital_ocean.go | 4 ++++ .../provider_digital_ocean_test.go | 3 +++ .../add_cloud_metadata/provider_google_gce.go | 6 +++++- .../provider_google_gce_test.go | 3 +++ .../provider_openstack_nova.go | 3 +++ .../provider_openstack_nova_test.go | 3 +++ .../provider_tencent_cloud.go | 3 +++ .../provider_tencent_cloud_test.go | 3 +++ 15 files changed, 63 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d6ed4bade49..36d8d0c9c3c2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -600,6 +600,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883] - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012] - Add `wineventlog` schema to `decode_xml` processor. {issue}23910[23910] {pull}24726[24726] +- Add new ECS 1.9 field `cloud.service.name` to `add_cloud_metadata` processor. {pull}24993[24993] *Auditbeat* diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go index 7d9e9ee986f5..65dd9c232868 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go @@ -33,6 +33,9 @@ var alibabaCloudMetadataFetcher = provider{ ecsMetadataZoneURI := "/latest/meta-data/zone-id" ecsSchema := func(m map[string]interface{}) common.MapStr { + m["service"] = common.MapStr{ + "name": "ECS", + } return common.MapStr(m) } diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go index 8d887245d86a..4adb41151b55 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -81,6 +81,9 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { }, "region": "cn-shenzhen", "availability_zone": "cn-shenzhen-a", + "service": common.MapStr{ + "name": "ECS", + }, }, } assert.Equal(t, expected, actual.Fields) diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go index b65d914f10c9..cbd825714681 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -33,13 +33,17 @@ var ec2MetadataFetcher = provider{ Create: func(_ string, config *common.Config) (metadataFetcher, error) { ec2Schema := func(m map[string]interface{}) common.MapStr { + m["serviceName"] = "EC2" out, _ := s.Schema{ "instance": s.Object{"id": c.Str("instanceId")}, "machine": s.Object{"type": c.Str("instanceType")}, "region": c.Str("region"), "availability_zone": c.Str("availabilityZone"), - "account": s.Object{"id": c.Str("accountId")}, - "image": s.Object{"id": c.Str("imageId")}, + "service": s.Object{ + "name": c.Str("serviceName"), + }, + "account": s.Object{"id": c.Str("accountId")}, + "image": s.Object{"id": c.Str("imageId")}, }.Apply(m) return out } diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go index 4a54e549c322..72053c673380 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go @@ -72,7 +72,7 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "imageId" : "%s", "instanceType" : "%s", "devpayProductCodes" : null, - "privateIp" : "10.0.0.1", + "privateIp" : "10.0.0.1", "version" : "2010-08-31", "billingProducts" : null, "pendingTime" : "2016-09-20T15:43:02Z", @@ -114,6 +114,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "image": common.MapStr{"id": imageIDDoc1}, "region": regionDoc1, "availability_zone": availabilityZoneDoc1, + "service": common.MapStr{ + "name": "EC2", + }, }, }, }, @@ -154,6 +157,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "image": common.MapStr{"id": imageIDDoc1}, "region": regionDoc1, "availability_zone": availabilityZoneDoc1, + "service": common.MapStr{ + "name": "EC2", + }, }, }, }, @@ -172,6 +178,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "image": common.MapStr{"id": imageIDDoc1}, "region": regionDoc1, "availability_zone": availabilityZoneDoc1, + "service": common.MapStr{ + "name": "EC2", + }, }, }, }, @@ -194,6 +203,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "image": common.MapStr{"id": imageIDDoc1}, "region": regionDoc1, "availability_zone": availabilityZoneDoc1, + "service": common.MapStr{ + "name": "EC2", + }, }, }, }, @@ -215,6 +227,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) { "image": common.MapStr{"id": imageIDDoc1}, "region": regionDoc1, "availability_zone": availabilityZoneDoc1, + "service": common.MapStr{ + "name": "EC2", + }, }, }, }, diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index 3028d531c1e5..b3f2a0b3222b 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -33,6 +33,7 @@ var azureVMMetadataFetcher = provider{ azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02" azHeaders := map[string]string{"Metadata": "true"} azSchema := func(m map[string]interface{}) common.MapStr { + m["serviceName"] = "Virtual Machines" out, _ := s.Schema{ "account": s.Object{ "id": c.Str("subscriptionId"), @@ -44,6 +45,9 @@ var azureVMMetadataFetcher = provider{ "machine": s.Object{ "type": c.Str("vmSize"), }, + "service": s.Object{ + "name": c.Str("serviceName"), + }, "region": c.Str("location"), }.Apply(m) return out diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index a988cc8873f8..5ebaad2c4a11 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -91,6 +91,9 @@ func TestRetrieveAzureMetadata(t *testing.T) { "account": common.MapStr{ "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e", }, + "service": common.MapStr{ + "name": "Virtual Machines", + }, "region": "eastus2", }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go index cc56ae044bd2..04da6228378f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go @@ -31,11 +31,15 @@ var doMetadataFetcher = provider{ Create: func(provider string, config *common.Config) (metadataFetcher, error) { doSchema := func(m map[string]interface{}) common.MapStr { + m["serviceName"] = "Droplets" out, _ := s.Schema{ "instance": s.Object{ "id": c.StrFromNum("droplet_id"), }, "region": c.Str("region"), + "service": s.Object{ + "name": c.Str("serviceName"), + }, }.Apply(m) return out } diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go index 5fb19a98feec..f39bafacc1a1 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go @@ -117,6 +117,9 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) { "instance": common.MapStr{ "id": "1111111", }, + "service": common.MapStr{ + "name": "Droplets", + }, "region": "nyc3", }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go index c17c1dfe2bd1..155e3a883d06 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce.go @@ -35,7 +35,11 @@ var gceMetadataFetcher = provider{ gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json" gceHeaders := map[string]string{"Metadata-Flavor": "Google"} gceSchema := func(m map[string]interface{}) common.MapStr { - out := common.MapStr{} + out := common.MapStr{ + "service": common.MapStr{ + "name": "GCE", + }, + } trimLeadingPath := func(key string) { v, err := out.GetValue(key) diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go index 0c810fe7a291..77ad27283a18 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go @@ -167,6 +167,9 @@ func TestRetrieveGCEMetadata(t *testing.T) { "project": common.MapStr{ "id": "test-dev", }, + "service": common.MapStr{ + "name": "GCE", + }, }, } assert.Equal(t, expected, actual.Fields) diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go index 7c9a997e0e23..01ada43cfb3e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go @@ -46,6 +46,9 @@ var openstackNovaSSLMetadataFetcher = provider{ func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Config) (metadataFetcher, error) { return func(provider string, c *common.Config) (metadataFetcher, error) { osSchema := func(m map[string]interface{}) common.MapStr { + m["service"] = common.MapStr{ + "name": "Nova", + } return common.MapStr(m) } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index 0a63c026cde3..31a4937343ac 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -109,6 +109,9 @@ func assertOpenstackNova(t *testing.T, config *common.Config) { "type": "m1.xlarge", }, "availability_zone": "az-test-2", + "service": common.MapStr{ + "name": "Nova", + }, }, } assert.Equal(t, expected, actual.Fields) diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go index be5956955353..0f09f4944ae3 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -33,6 +33,9 @@ var qcloudMetadataFetcher = provider{ qcloudMetadataZoneURI := "/meta-data/placement/zone" qcloudSchema := func(m map[string]interface{}) common.MapStr { + m["service"] = common.MapStr{ + "name": "CVM", + } return common.MapStr(m) } diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go index 1615d37a38cc..5959bf57aef6 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -81,6 +81,9 @@ func TestRetrieveQCloudMetadata(t *testing.T) { }, "region": "china-south-gz", "availability_zone": "gz-azone2", + "service": common.MapStr{ + "name": "CVM", + }, }, } assert.Equal(t, expected, actual.Fields) From 7b729da14738a5156ca62e8d8e47e3389f6bc18c Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 9 Apr 2021 09:33:59 -0500 Subject: [PATCH 12/12] [Filebeat] Fix hardcoded amazonaws.com endpoint (#24861) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/awss3/collector.go | 16 +++-- x-pack/filebeat/input/awss3/collector_test.go | 66 +++++++++++++++++-- x-pack/filebeat/input/awss3/input.go | 2 +- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 36d8d0c9c3c2..90682d343781 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -387,6 +387,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity. Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766] - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829] - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] +- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go index 806bead57b26..d86220faf7da 100644 --- a/x-pack/filebeat/input/awss3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -226,14 +226,20 @@ func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout return err } -func getRegionFromQueueURL(queueURL string) (string, error) { +func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) { // get region from queueURL // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs - queueURLSplit := strings.Split(queueURL, ".") - if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { - return queueURLSplit[1], nil + url, err := url.Parse(queueURL) + if err != nil { + return "", fmt.Errorf(queueURL + " is not a valid URL") + } + if url.Scheme == "https" && url.Host != "" { + queueHostSplit := strings.Split(url.Host, ".") + if len(queueHostSplit) > 2 && (strings.Join(queueHostSplit[2:], ".") == endpoint || (endpoint == "" && queueHostSplit[2] == "amazonaws")) { + return queueHostSplit[1], nil + } } - return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") + return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") } // handle message diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go index fa613e29df89..e2a8bfca53e5 100644 --- a/x-pack/filebeat/input/awss3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -57,10 +57,68 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe } func TestGetRegionFromQueueURL(t *testing.T) { - queueURL := "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs" - regionName, err := getRegionFromQueueURL(queueURL) - assert.NoError(t, err) - assert.Equal(t, "us-east-1", regionName) + casesPositive := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using amazonaws.com domain with blank Endpoint", + "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + "", + "us-east-1", + }, + { + "QueueURL using abc.xyz and domain with matching Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "abc.xyz", + "us-east-1", + }, + } + + for _, c := range casesPositive { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.NoError(t, err) + assert.Equal(t, c.expectedRegion, regionName) + }) + } + + casesNegative := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using abc.xyz and domain with blank Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "", + "", + }, + { + "QueueURL using abc.xyz and domain with different Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "googlecloud.com", + "", + }, + { + "QueueURL is an invalid URL", + ":foo", + "", + "", + }, + } + + for _, c := range casesNegative { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.Error(t, err) + assert.Empty(t, regionName) + }) + } + } func TestHandleMessage(t *testing.T) { diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index ccbe105974d8..bdb5976bf2da 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -87,7 +87,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C return nil, err } - regionName, err := getRegionFromQueueURL(in.config.QueueURL) + regionName, err := getRegionFromQueueURL(in.config.QueueURL, in.config.AwsConfig.Endpoint) if err != nil { err := fmt.Errorf("getRegionFromQueueURL failed: %w", err) log.Error(err)