stevesg/hairyhenderson otel tracing support

.gitignore

@@ -21,3 +21,7 @@
 !/.promu.yml
 !/api/v2/openapi.yaml
 !.github/workflows/*.yml
+
+# Editor
+.vscode
+.DS_Store

api/v2/api.go

@@ -33,7 +33,6 @@ import (
 	"github.com/prometheus/common/version"
 	"github.com/rs/cors"
 
-	"github.com/prometheus/alertmanager/api/metrics"
 	open_api_models "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/alertmanager/api/v2/restapi"
 	"github.com/prometheus/alertmanager/api/v2/restapi/operations"
@@ -42,6 +41,8 @@ import (
 	general_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/general"
 	receiver_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver"
 	silence_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/silence"
+
+	"github.com/prometheus/alertmanager/api/metrics"
 	"github.com/prometheus/alertmanager/cluster"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/dispatch"
@@ -225,8 +226,9 @@ func (api *API) getReceiversHandler(params receiver_ops.GetReceiversParams) midd
 	defer api.mtx.RUnlock()
 
 	receivers := make([]*open_api_models.Receiver, 0, len(api.alertmanagerConfig.Receivers))
-	for i := range api.alertmanagerConfig.Receivers {
-		receivers = append(receivers, &open_api_models.Receiver{Name: &api.alertmanagerConfig.Receivers[i].Name})
+	for _, r := range api.alertmanagerConfig.Receivers {
+		name := r.Name
+		receivers = append(receivers, &open_api_models.Receiver{Name: &name})
 	}
 
 	return receiver_ops.NewGetReceiversOK().WithPayload(receivers)
@@ -660,8 +662,7 @@ func (api *API) postSilencesHandler(params silence_ops.PostSilencesParams) middl
 		return silence_ops.NewPostSilencesBadRequest().WithPayload(msg)
 	}
 
-	sid, err := api.silences.Set(sil)
-	if err != nil {
+	if err = api.silences.Set(sil); err != nil {
 		level.Error(logger).Log("msg", "Failed to create silence", "err", err)
 		if errors.Is(err, silence.ErrNotFound) {
 			return silence_ops.NewPostSilencesNotFound().WithPayload(err.Error())
@@ -670,7 +671,7 @@ func (api *API) postSilencesHandler(params silence_ops.PostSilencesParams) middl
 	}
 
 	return silence_ops.NewPostSilencesOK().WithPayload(&silence_ops.PostSilencesOKBody{
-		SilenceID: sid,
+		SilenceID: sil.Id,
 	})
 }
 

api/v2/api_test.go

@@ -15,6 +15,7 @@ package v2
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -24,6 +25,7 @@ import (
 	"time"
 
 	"github.com/go-openapi/runtime"
+	"github.com/go-openapi/runtime/middleware"
 	"github.com/go-openapi/strfmt"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
@@ -159,18 +161,16 @@ func TestDeleteSilenceHandler(t *testing.T) {
 		EndsAt:    now.Add(time.Hour),
 		UpdatedAt: now,
 	}
-	unexpiredSid, err := silences.Set(unexpiredSil)
-	require.NoError(t, err)
+	require.NoError(t, silences.Set(unexpiredSil))
 
 	expiredSil := &silencepb.Silence{
 		Matchers:  []*silencepb.Matcher{m},
 		StartsAt:  now.Add(-time.Hour),
 		EndsAt:    now.Add(time.Hour),
 		UpdatedAt: now,
 	}
-	expiredSid, err := silences.Set(expiredSil)
-	require.NoError(t, err)
-	require.NoError(t, silences.Expire(expiredSid))
+	require.NoError(t, silences.Set(expiredSil))
+	require.NoError(t, silences.Expire(expiredSil.Id))
 
 	for i, tc := range []struct {
 		sid          string
@@ -181,11 +181,11 @@ func TestDeleteSilenceHandler(t *testing.T) {
 			404,
 		},
 		{
-			unexpiredSid,
+			unexpiredSil.Id,
 			200,
 		},
 		{
-			expiredSid,
+			expiredSil.Id,
 			200,
 		},
 	} {
@@ -223,18 +223,16 @@ func TestPostSilencesHandler(t *testing.T) {
 		EndsAt:    now.Add(time.Hour),
 		UpdatedAt: now,
 	}
-	unexpiredSid, err := silences.Set(unexpiredSil)
-	require.NoError(t, err)
+	require.NoError(t, silences.Set(unexpiredSil))
 
 	expiredSil := &silencepb.Silence{
 		Matchers:  []*silencepb.Matcher{m},
 		StartsAt:  now.Add(-time.Hour),
 		EndsAt:    now.Add(time.Hour),
 		UpdatedAt: now,
 	}
-	expiredSid, err := silences.Set(expiredSil)
-	require.NoError(t, err)
-	require.NoError(t, silences.Expire(expiredSid))
+	require.NoError(t, silences.Set(expiredSil))
+	require.NoError(t, silences.Expire(expiredSil.Id))
 
 	t.Run("Silences CRUD", func(t *testing.T) {
 		for i, tc := range []struct {
@@ -259,46 +257,122 @@ func TestPostSilencesHandler(t *testing.T) {
 			},
 			{
 				"with an active silence ID - it extends the silence",
-				unexpiredSid,
+				unexpiredSil.Id,
 				now.Add(time.Hour),
 				now.Add(time.Hour * 2),
 				200,
 			},
 			{
 				"with an expired silence ID - it re-creates the silence",
-				expiredSid,
+				expiredSil.Id,
 				now.Add(time.Hour),
 				now.Add(time.Hour * 2),
 				200,
 			},
 		} {
 			t.Run(tc.name, func(t *testing.T) {
-				silence, silenceBytes := createSilence(t, tc.sid, "silenceCreator", tc.start, tc.end)
-
 				api := API{
 					uptime:   time.Now(),
 					silences: silences,
 					logger:   log.NewNopLogger(),
 				}
 
-				r, err := http.NewRequest("POST", "/api/v2/silence/${tc.sid}", bytes.NewReader(silenceBytes))
-				require.NoError(t, err)
-
+				sil := createSilence(t, tc.sid, "silenceCreator", tc.start, tc.end)
 				w := httptest.NewRecorder()
-				p := runtime.TextProducer()
-				responder := api.postSilencesHandler(silence_ops.PostSilencesParams{
-					HTTPRequest: r,
-					Silence:     &silence,
-				})
-				responder.WriteResponse(w, p)
+				postSilences(t, w, api.postSilencesHandler, sil)
 				body, _ := io.ReadAll(w.Result().Body)
-
 				require.Equal(t, tc.expectedCode, w.Code, fmt.Sprintf("test case: %d, response: %s", i, string(body)))
 			})
 		}
 	})
 }
 
+func TestPostSilencesHandlerMissingIdCreatesSilence(t *testing.T) {
+	now := time.Now()
+	silences := newSilences(t)
+	api := API{
+		uptime:   time.Now(),
+		silences: silences,
+		logger:   log.NewNopLogger(),
+	}
+
+	// Create a new silence. It should be assigned a random UUID.
+	sil := createSilence(t, "", "silenceCreator", now.Add(time.Hour), now.Add(time.Hour*2))
+	w := httptest.NewRecorder()
+	postSilences(t, w, api.postSilencesHandler, sil)
+	require.Equal(t, http.StatusOK, w.Code)
+
+	// Get the silences from the API.
+	w = httptest.NewRecorder()
+	getSilences(t, w, api.getSilencesHandler)
+	require.Equal(t, http.StatusOK, w.Code)
+	var resp []open_api_models.GettableSilence
+	require.NoError(t, json.NewDecoder(w.Body).Decode(&resp))
+	require.Len(t, resp, 1)
+
+	// Change the ID. It should return 404 Not Found.
+	sil = open_api_models.PostableSilence{
+		ID:      "unknownID",
+		Silence: resp[0].Silence,
+	}
+	w = httptest.NewRecorder()
+	postSilences(t, w, api.postSilencesHandler, sil)
+	require.Equal(t, http.StatusNotFound, w.Code)
+
+	// Remove the ID. It should duplicate the silence with a different UUID.
+	sil = open_api_models.PostableSilence{
+		ID:      "",
+		Silence: resp[0].Silence,
+	}
+	w = httptest.NewRecorder()
+	postSilences(t, w, api.postSilencesHandler, sil)
+	require.Equal(t, http.StatusOK, w.Code)
+
+	// Get the silences from the API. There should now be 2 silences.
+	w = httptest.NewRecorder()
+	getSilences(t, w, api.getSilencesHandler)
+	require.Equal(t, http.StatusOK, w.Code)
+	require.NoError(t, json.NewDecoder(w.Body).Decode(&resp))
+	require.Len(t, resp, 2)
+	require.NotEqual(t, resp[0].ID, resp[1].ID)
+}
+
+func getSilences(
+	t *testing.T,
+	w *httptest.ResponseRecorder,
+	handlerFunc func(params silence_ops.GetSilencesParams) middleware.Responder,
+) {
+	r, err := http.NewRequest("GET", "/api/v2/silences", nil)
+	require.NoError(t, err)
+
+	p := runtime.TextProducer()
+	responder := handlerFunc(silence_ops.GetSilencesParams{
+		HTTPRequest: r,
+		Filter:      nil,
+	})
+	responder.WriteResponse(w, p)
+}
+
+func postSilences(
+	t *testing.T,
+	w *httptest.ResponseRecorder,
+	handlerFunc func(params silence_ops.PostSilencesParams) middleware.Responder,
+	sil open_api_models.PostableSilence,
+) {
+	b, err := json.Marshal(sil)
+	require.NoError(t, err)
+
+	r, err := http.NewRequest("POST", "/api/v2/silences", bytes.NewReader(b))
+	require.NoError(t, err)
+
+	p := runtime.TextProducer()
+	responder := handlerFunc(silence_ops.PostSilencesParams{
+		HTTPRequest: r,
+		Silence:     &sil,
+	})
+	responder.WriteResponse(w, p)
+}
+
 func TestCheckSilenceMatchesFilterLabels(t *testing.T) {
 	type test struct {
 		silenceMatchers []*silencepb.Matcher

api/v2/testing.go

@@ -14,19 +14,17 @@
 package v2
 
 import (
-	"encoding/json"
 	"testing"
 	"time"
 
 	"github.com/go-openapi/strfmt"
-	"github.com/stretchr/testify/require"
 
 	open_api_models "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/alertmanager/silence/silencepb"
 )
 
-func createSilence(t *testing.T, ID, creator string, start, ends time.Time) (open_api_models.PostableSilence, []byte) {
+func createSilence(t *testing.T, ID, creator string, start, ends time.Time) open_api_models.PostableSilence {
 	t.Helper()
 
 	comment := "test"
@@ -46,10 +44,7 @@ func createSilence(t *testing.T, ID, creator string, start, ends time.Time) (ope
 			Comment:   &comment,
 		},
 	}
-	b, err := json.Marshal(&sil)
-	require.NoError(t, err)
-
-	return sil, b
+	return sil
 }
 
 func createSilenceMatcher(t *testing.T, name, pattern string, matcherType silencepb.Matcher_Type) *silencepb.Matcher {

cmd/alertmanager/main.go

@@ -145,6 +145,8 @@ func run() int {
 		dataDir             = kingpin.Flag("storage.path", "Base path for data storage.").Default("data/").String()
 		retention           = kingpin.Flag("data.retention", "How long to keep data for.").Default("120h").Duration()
 		maintenanceInterval = kingpin.Flag("data.maintenance-interval", "Interval between garbage collection and snapshotting to disk of the silences and the notification logs.").Default("15m").Duration()
+		maxSilences         = kingpin.Flag("silences.max-silences", "Maximum number of silences, including expired silences. If negative or zero, no limit is set.").Default("0").Int()
+		maxSilenceSizeBytes = kingpin.Flag("silences.max-silence-size-bytes", "Maximum silence size in bytes. If negative or zero, no limit is set.").Default("0").Int()
 		alertGCInterval     = kingpin.Flag("alerts.gc-interval", "Interval between alert GC.").Default("30m").Duration()
 
 		webConfig      = webflag.AddFlags(kingpin.CommandLine, ":9093")
@@ -258,8 +260,12 @@ func run() int {
 	silenceOpts := silence.Options{
 		SnapshotFile: filepath.Join(*dataDir, "silences"),
 		Retention:    *retention,
-		Logger:       log.With(logger, "component", "silences"),
-		Metrics:      prometheus.DefaultRegisterer,
+		Limits: silence.Limits{
+			MaxSilences:         func() int { return *maxSilences },
+			MaxSilenceSizeBytes: func() int { return *maxSilenceSizeBytes },
+		},
+		Logger:  log.With(logger, "component", "silences"),
+		Metrics: prometheus.DefaultRegisterer,
 	}
 
 	silences, err := silence.New(silenceOpts)
@@ -383,16 +389,16 @@ func run() int {
 
 		// Build the routing tree and record which receivers are used.
 		routes := dispatch.NewRoute(conf.Route, nil)
-		activeReceivers := make(map[string]struct{})
+		activeReceiversMap := make(map[string]struct{})
 		routes.Walk(func(r *dispatch.Route) {
-			activeReceivers[r.RouteOpts.Receiver] = struct{}{}
+			activeReceiversMap[r.RouteOpts.Receiver] = struct{}{}
 		})
 
 		// Build the map of receiver to integrations.
-		receivers := make(map[string][]notify.Integration, len(activeReceivers))
+		receivers := make(map[string][]*notify.Integration, len(activeReceiversMap))
 		var integrationsNum int
 		for _, rcv := range conf.Receivers {
-			if _, found := activeReceivers[rcv.Name]; !found {
+			if _, found := activeReceiversMap[rcv.Name]; !found {
 				// No need to build a receiver if no route is using it.
 				level.Info(configLogger).Log("msg", "skipping creation of receiver not referenced by any route", "receiver", rcv.Name)
 				continue
@@ -442,7 +448,7 @@ func run() int {
 			pipelinePeer,
 		)
 
-		configuredReceivers.Set(float64(len(activeReceivers)))
+		configuredReceivers.Set(float64(len(activeReceiversMap)))
 		configuredIntegrations.Set(float64(integrationsNum))
 		configuredInhibitionRules.Set(float64(len(conf.InhibitRules)))
 

config/receiver/receiver.go

@@ -39,10 +39,10 @@ import (
 
 // BuildReceiverIntegrations builds a list of integration notifiers off of a
 // receiver config.
-func BuildReceiverIntegrations(nc config.Receiver, tmpl *template.Template, logger log.Logger, httpOpts ...commoncfg.HTTPClientOption) ([]notify.Integration, error) {
+func BuildReceiverIntegrations(nc config.Receiver, tmpl *template.Template, logger log.Logger, httpOpts ...commoncfg.HTTPClientOption) ([]*notify.Integration, error) {
 	var (
 		errs         types.MultiError
-		integrations []notify.Integration
+		integrations []*notify.Integration
 		add          = func(name string, i int, rs notify.ResolvedSender, f func(l log.Logger) (notify.Notifier, error)) {
 			n, err := f(log.With(logger, "integration", name))
 			if err != nil {

config/receiver/receiver_test.go

@@ -31,7 +31,7 @@ func TestBuildReceiverIntegrations(t *testing.T) {
 	for _, tc := range []struct {
 		receiver config.Receiver
 		err      bool
-		exp      []notify.Integration
+		exp      []*notify.Integration
 	}{
 		{
 			receiver: config.Receiver{
@@ -48,7 +48,7 @@ func TestBuildReceiverIntegrations(t *testing.T) {
 					},
 				},
 			},
-			exp: []notify.Integration{
+			exp: []*notify.Integration{
 				notify.NewIntegration(nil, sendResolved(false), "webhook", 0, "foo"),
 				notify.NewIntegration(nil, sendResolved(true), "webhook", 1, "foo"),
 			},