Add /api/fleet/agents/:id/audit/unenroll endpoint (elastic#3818)

Add /api/fleet/agents/:id/audit/unenroll API that an elastic-agent or Endpoint process may use to annotate the agent document so the agent may appear with a different status.

changelog/fragments/1723590385-Add-audit-unenroll-API.yaml

@@ -0,0 +1,36 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add audit/unenroll API
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Add the /api/fleet/agents/:id/audit/unenroll API that elastic-agent
+  and Endpoint instances may use to annotate the agent document when
+  the agent is uninstalled or Endpoint detects it is in an orphaned
+  state.
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/fleet-server/pull/3818
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/484

fleet-server.reference.yml

@@ -222,6 +222,11 @@ fleet:
 #         burst: 100
 #         max: 50
 #         max_body_byte_size: 0
+#       audit_unenroll_limit:
+#         interval: 10ms
+#         burst: 100
+#         max: 50
+#         max_body_byte_size: 1024
 #
 #     # go runtime limits
 #     runtime:

internal/pkg/api/api.go

@@ -28,6 +28,7 @@ type apiServer struct {
 	ut     *UploadT
 	ft     *FileDeliveryT
 	pt     *PGPRetrieverT
+	audit  *AuditT
 	bulker bulk.Bulk
 }
 
@@ -148,6 +149,15 @@ func (a *apiServer) GetPGPKey(w http.ResponseWriter, r *http.Request, major, min
 	}
 }
 
+func (a *apiServer) AuditUnenroll(w http.ResponseWriter, r *http.Request, id string, params AuditUnenrollParams) {
+	zlog := hlog.FromRequest(r).With().Str(LogAgentID, id).Logger()
+	if err := a.audit.handleUnenroll(zlog, w, r, id); err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		cntAuditUnenroll.IncError(err)
+		ErrorResp(w, r, err)
+	}
+}
+
 func (a *apiServer) Status(w http.ResponseWriter, r *http.Request, params StatusParams) {
 	zlog := hlog.FromRequest(r).With().
 		Str("mod", kStatusMod).

internal/pkg/api/error.go

@@ -481,6 +481,16 @@ func NewHTTPErrResp(err error) HTTPErrResp {
 				zerolog.InfoLevel,
 			},
 		},
+		// audit unenroll
+		{
+			ErrAuditUnenrollReason,
+			HTTPErrResp{
+				http.StatusConflict,
+				"ErrAuditReasonConflict",
+				"agent document contains audit_unenroll_reason",
+				zerolog.InfoLevel,
+			},
+		},
 	}
 
 	for _, e := range errTable {

internal/pkg/api/handleAudit.go

@@ -0,0 +1,122 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
+	"github.com/elastic/fleet-server/v7/internal/pkg/cache"
+	"github.com/elastic/fleet-server/v7/internal/pkg/config"
+	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
+	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+
+	"github.com/miolini/datacounter"
+	"github.com/rs/zerolog"
+	"go.elastic.co/apm/v2"
+)
+
+var ErrAuditUnenrollReason = fmt.Errorf("agent document contains audit_unenroll_reason attribute")
+
+type AuditT struct {
+	cfg   *config.Server
+	bulk  bulk.Bulk
+	cache cache.Cache
+}
+
+func NewAuditT(cfg *config.Server, bulker bulk.Bulk, cache cache.Cache) *AuditT {
+	return &AuditT{
+		cfg:   cfg,
+		bulk:  bulker,
+		cache: cache,
+	}
+}
+
+func (audit *AuditT) handleUnenroll(zlog zerolog.Logger, w http.ResponseWriter, r *http.Request, id string) error {
+	agent, err := authAgent(r, &id, audit.bulk, audit.cache)
+	if err != nil {
+		return err
+	}
+	zlog = zlog.With().Str(LogAccessAPIKeyID, agent.AccessAPIKeyID).Logger()
+	ctx := zlog.WithContext(r.Context())
+	r = r.WithContext(ctx)
+
+	return audit.unenroll(zlog, w, r, agent)
+}
+
+func (audit *AuditT) unenroll(zlog zerolog.Logger, w http.ResponseWriter, r *http.Request, agent *model.Agent) error {
+	if agent.AuditUnenrolledReason != "" {
+		return ErrAuditUnenrollReason
+	}
+
+	req, err := audit.validateUnenrollRequest(zlog, w, r)
+	if err != nil {
+		return err
+	}
+
+	if err := audit.markUnenroll(r.Context(), zlog, req, agent); err != nil {
+		return err
+	}
+
+	span, _ := apm.StartSpan(r.Context(), "response", "write")
+	defer span.End()
+	w.WriteHeader(http.StatusOK)
+	return nil
+}
+
+func (audit *AuditT) validateUnenrollRequest(zlog zerolog.Logger, w http.ResponseWriter, r *http.Request) (*AuditUnenrollRequest, error) {
+	span, _ := apm.StartSpan(r.Context(), "validateRequest", "validate")
+	defer span.End()
+
+	body := r.Body
+	if audit.cfg.Limits.AuditUnenrollLimit.MaxBody > 0 {
+		body = http.MaxBytesReader(w, body, audit.cfg.Limits.AuditUnenrollLimit.MaxBody)
+	}
+	readCounter := datacounter.NewReaderCounter(body)
+
+	var req AuditUnenrollRequest
+	dec := json.NewDecoder(readCounter)
+	if err := dec.Decode(&req); err != nil {
+		return nil, &BadRequestErr{msg: "unable to decode audit/unenroll request", nextErr: err}
+	}
+
+	switch req.Reason {
+	case Uninstall, Orphaned, KeyRevoked:
+	default:
+		return nil, &BadRequestErr{msg: "audit/unenroll request invalid reason"}
+	}
+
+	cntAuditUnenroll.bodyIn.Add(readCounter.Count())
+	zlog.Trace().Msg("Audit unenroll request")
+	return &req, nil
+}
+
+func (audit *AuditT) markUnenroll(ctx context.Context, zlog zerolog.Logger, req *AuditUnenrollRequest, agent *model.Agent) error {
+	span, ctx := apm.StartSpan(ctx, "auditUnenroll", "process")
+	defer span.End()
+
+	now := time.Now().UTC().Format(time.RFC3339)
+	doc := bulk.UpdateFields{
+		dl.FieldUnenrolledAt:          now,
+		dl.FieldUpdatedAt:             now,
+		dl.FieldAuditUnenrolledTime:   req.Timestamp,
+		dl.FieldAuditUnenrolledReason: req.Reason,
+	}
+	body, err := doc.Marshal()
+	if err != nil {
+		return fmt.Errorf("auditUnenroll marshal: %w", err)
+	}
+
+	if err := audit.bulk.Update(ctx, dl.FleetAgents, agent.Id, body, bulk.WithRefresh(), bulk.WithRetryOnConflict(3)); err != nil {
+		return fmt.Errorf("auditUnenroll update: %w", err)
+	}
+
+	zlog.Info().Msg("audit unenroll successful")
+	return nil
+}

internal/pkg/api/handleAudit_test.go

@@ -0,0 +1,136 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package api
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/elastic/fleet-server/v7/internal/pkg/config"
+	"github.com/elastic/fleet-server/v7/internal/pkg/dl"
+	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
+	testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Audit_validateUnenrollRequst(t *testing.T) {
+	tests := []struct {
+		name  string
+		req   *http.Request
+		cfg   *config.Server
+		valid *AuditUnenrollRequest
+		err   error
+	}{{
+		name: "ok",
+		req: &http.Request{
+			Body: io.NopCloser(strings.NewReader(`{"reason":"uninstall", "timestamp": "2024-01-01T12:00:00.000Z"}`)),
+		},
+		cfg: &config.Server{},
+		valid: &AuditUnenrollRequest{
+			Reason:    Uninstall,
+			Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+		},
+		err: nil,
+	}, {
+		name: "not json object",
+		req: &http.Request{
+			Body: io.NopCloser(strings.NewReader(`{"invalidJson":}`)),
+		},
+		cfg:   &config.Server{},
+		valid: nil,
+		err:   &BadRequestErr{msg: "unable to decode audit/unenroll request"},
+	}, {
+		name: "bad reason",
+		req: &http.Request{
+			Body: io.NopCloser(strings.NewReader(`{"reason":"bad reason", "timestamp": "2024-01-01T12:00:00.000Z"}`)),
+		},
+		cfg:   &config.Server{},
+		valid: nil,
+		err:   &BadRequestErr{msg: "audit/unenroll request invalid reason"},
+	}, {
+		name: "too large",
+		req: &http.Request{
+			Body: io.NopCloser(strings.NewReader(`{"reason":"uninstalled", "timestamp": "2024-01-01T12:00:00.000Z"}`)),
+		},
+		cfg: &config.Server{
+			Limits: config.ServerLimits{
+				AuditUnenrollLimit: config.Limit{
+					MaxBody: 10,
+				},
+			},
+		},
+		valid: nil,
+		err:   &BadRequestErr{msg: "unable to decode audit/unenroll request"},
+	}}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			audit := AuditT{cfg: tc.cfg}
+			w := httptest.NewRecorder()
+
+			r, err := audit.validateUnenrollRequest(testlog.SetLogger(t), w, tc.req)
+			if tc.err != nil {
+				require.EqualError(t, err, tc.err.Error())
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, tc.valid, r)
+		})
+	}
+}
+
+func Test_Audit_markUnenroll(t *testing.T) {
+	agent := &model.Agent{
+		ESDocument: model.ESDocument{
+			Id: "test-id",
+		},
+	}
+	bulker := ftesting.NewMockBulk()
+	bulker.On("Update", mock.Anything, dl.FleetAgents, agent.Id, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	audit := AuditT{bulk: bulker}
+	logger := testlog.SetLogger(t)
+	err := audit.markUnenroll(context.Background(), logger, &AuditUnenrollRequest{Reason: Uninstall, Timestamp: time.Now().UTC()}, agent)
+	require.NoError(t, err)
+	bulker.AssertExpectations(t)
+}
+
+func Test_Audit_unenroll(t *testing.T) {
+	t.Run("agent has audit_unenroll_reason", func(t *testing.T) {
+		agent := &model.Agent{
+			AuditUnenrolledReason: string(Uninstall),
+		}
+		audit := &AuditT{}
+		err := audit.unenroll(testlog.SetLogger(t), nil, nil, agent)
+		require.EqualError(t, err, ErrAuditUnenrollReason.Error())
+	})
+
+	t.Run("ok", func(t *testing.T) {
+		agent := &model.Agent{
+			ESDocument: model.ESDocument{
+				Id: "test-id",
+			},
+		}
+		bulker := ftesting.NewMockBulk()
+		bulker.On("Update", mock.Anything, dl.FleetAgents, agent.Id, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+
+		audit := &AuditT{
+			bulk: bulker,
+			cfg:  &config.Server{},
+		}
+		req := &http.Request{
+			Body: io.NopCloser(strings.NewReader(`{"reason": "uninstall", "timestamp": "2024-01-01T12:00:00.000Z"}`)),
+		}
+		err := audit.unenroll(testlog.SetLogger(t), httptest.NewRecorder(), req, agent)
+		require.NoError(t, err)
+		bulker.AssertExpectations(t)
+	})
+}

internal/pkg/api/metrics.go

@@ -35,16 +35,17 @@ var (
 	cntHTTPClose  *statsCounter
 	cntHTTPActive *statsGauge
 
-	cntCheckin     routeStats
-	cntEnroll      routeStats
-	cntAcks        routeStats
-	cntStatus      routeStats
-	cntUploadStart routeStats
-	cntUploadChunk routeStats
-	cntUploadEnd   routeStats
-	cntFileDeliv   routeStats
-	cntGetPGP      routeStats
-	cntArtifacts   artifactStats
+	cntCheckin       routeStats
+	cntEnroll        routeStats
+	cntAcks          routeStats
+	cntStatus        routeStats
+	cntUploadStart   routeStats
+	cntUploadChunk   routeStats
+	cntUploadEnd     routeStats
+	cntFileDeliv     routeStats
+	cntGetPGP        routeStats
+	cntAuditUnenroll routeStats
+	cntArtifacts     artifactStats
 
 	infoReg sync.Once
 )
@@ -75,7 +76,7 @@ func init() {
 	cntUploadEnd.Register(routesRegistry.newRegistry("uploadEnd"))
 	cntFileDeliv.Register(routesRegistry.newRegistry("deliverFile"))
 	cntGetPGP.Register(routesRegistry.newRegistry("getPGPKey"))
-
+	cntAuditUnenroll.Register(routesRegistry.newRegistry("auditUnenroll"))
 }
 
 // metricsRegistry wraps libbeat and prometheus registries