Add request id to HTTP transactions to allow tracking. (#28590) (#29324)

* Add request id to HTTP transactions to allow tracking through the proxy into Fleet Server and back.

* Linter fix

(cherry picked from commit 6288ccc)

Co-authored-by: Sean Cunningham <[email protected]>

x-pack/elastic-agent/pkg/remote/client.go

@@ -20,6 +20,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common/transport/httpcommon"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/id"
 )
 
 const (
@@ -166,7 +167,13 @@ func (c *Client) Send(
 	headers http.Header,
 	body io.Reader,
 ) (*http.Response, error) {
-	c.log.Debugf("Request method: %s, path: %s", method, path)
+	// Generate a request ID for tracking
+	var reqID string
+	if u, err := id.Generate(); err == nil {
+		reqID = u.String()
+	}
+
+	c.log.Debugf("Request method: %s, path: %s, reqID: %s", method, path, reqID)
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	requester := c.nextRequester()
@@ -183,6 +190,11 @@ func (c *Client) Send(
 	// TODO: Make this header specific to fleet-server or remove it
 	req.Header.Set("kbn-xsrf", "1") // Without this Kibana will refuse to answer the request.
 
+	// If available, add the request id as an HTTP header
+	if reqID != "" {
+		req.Header.Add("X-Request-ID", reqID)
+	}
+
 	// copy headers.
 	for header, values := range headers {
 		for _, v := range values {

x-pack/elastic-agent/pkg/remote/client_test.go

@@ -279,6 +279,33 @@ func TestHTTPClient(t *testing.T) {
 			assert.Equal(t, 1, len(debugger.messages))
 		},
 	))
+
+	t.Run("RequestId", withServer(
+		func(t *testing.T) *http.ServeMux {
+			msg := `{ message: "hello" }`
+			mux := http.NewServeMux()
+			mux.HandleFunc("/echo-hello", func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				fmt.Fprint(w, msg)
+				require.NotEmpty(t, r.Header.Get("X-Request-ID"))
+			})
+			return mux
+		}, func(t *testing.T, host string) {
+			cfg := config.MustNewConfigFrom(map[string]interface{}{
+				"host": host,
+			})
+
+			client, err := NewWithRawConfig(nil, cfg, nil)
+			require.NoError(t, err)
+			resp, err := client.Send(ctx, "GET", "/echo-hello", nil, nil, nil)
+			require.NoError(t, err)
+
+			body, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, `{ message: "hello" }`, string(body))
+		},
+	))
 }
 
 func TestNextRequester(t *testing.T) {