Start metric and status probe server as soon as possible

Signed-off-by: Kemal Akkoyun <[email protected]>
thanos-io · Oct 16, 2019 · 740f37f · 740f37f
1 parent fb0db63
commit 740f37f
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 80 deletions.
diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -173,6 +173,7 @@ func runCompact(
 
 	downsampleMetrics := newDownsampleMetrics(reg)
 
+	level.Debug(logger).Log("msg", "setting up http server")
 	statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
 	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil {

diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go
@@ -95,7 +95,13 @@ func runDownsample(
 	}()
 
 	metrics := newDownsampleMetrics(reg)
+	level.Debug(logger).Log("msg", "setting up http server")
 	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {
+		return errors.Wrap(err, "schedule HTTP server with probe")
+	}
+
 	// Start cycle of syncing blocks from the bucket and garbage collecting the bucket.
 	{
 		ctx, cancel := context.WithCancel(context.Background())
@@ -122,11 +128,6 @@ func runDownsample(
 		})
 	}
 
-	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
-	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {
-		return errors.Wrap(err, "schedule HTTP server with probe")
-	}
-
 	level.Info(logger).Log("msg", "starting downsample node")
 	return nil
 }

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -342,6 +342,37 @@ func runQuery(
 			stores.Close()
 		})
 	}
+	// Start query API + UI, metrics and status probe HTTP server.
+	level.Debug(logger).Log("msg", "setting up http server")
+	statusProber := prober.NewProber(comp, logger, reg)
+	{
+		router := route.New()
+
+		// Redirect from / to /webRoutePrefix.
+		if webRoutePrefix != "" {
+			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
+				http.Redirect(w, r, webRoutePrefix, http.StatusFound)
+			})
+		}
+
+		flagsMap := map[string]string{
+			// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
+			"web.external-prefix": webExternalPrefix,
+			"web.prefix-header":   webPrefixHeaderName,
+		}
+
+		ins := extpromhttp.NewInstrumentationMiddleware(reg)
+		ui.NewQueryUI(logger, reg, stores, flagsMap).Register(router.WithPrefix(webRoutePrefix), ins)
+
+		api := v1.NewAPI(logger, reg, engine, queryableCreator, enableAutodownsampling, enablePartialResponse, replicaLabels, instantDefaultMaxSourceResolution)
+
+		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
+
+		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
+			return errors.Wrap(err, "schedule HTTP server with probes")
+		}
+	}
 	// Run File Service Discovery and update the store set when the files are modified.
 	if fileSD != nil {
 		var fileSDUpdates chan []*targetgroup.Group
@@ -389,37 +420,6 @@ func runQuery(
 			cancel()
 		})
 	}
-	// Start query API + UI HTTP server.
-
-	statusProber := prober.NewProber(comp, logger, reg)
-	{
-		router := route.New()
-
-		// Redirect from / to /webRoutePrefix.
-		if webRoutePrefix != "" {
-			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-				http.Redirect(w, r, webRoutePrefix, http.StatusFound)
-			})
-		}
-
-		flagsMap := map[string]string{
-			// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
-			"web.external-prefix": webExternalPrefix,
-			"web.prefix-header":   webPrefixHeaderName,
-		}
-
-		ins := extpromhttp.NewInstrumentationMiddleware(reg)
-		ui.NewQueryUI(logger, reg, stores, flagsMap).Register(router.WithPrefix(webRoutePrefix), ins)
-
-		api := v1.NewAPI(logger, reg, engine, queryableCreator, enableAutodownsampling, enablePartialResponse, replicaLabels, instantDefaultMaxSourceResolution)
-
-		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
-
-		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
-		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
-			return errors.Wrap(err, "schedule HTTP server with probes")
-		}
-	}
 	// Start query (proxy) gRPC StoreAPI.
 	{
 		l, err := net.Listen("tcp", grpcBindAddr)

diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -163,7 +163,13 @@ func runReceive(
 		Tracer:            tracer,
 	})
 
+	level.Debug(logger).Log("msg", "setting up http server")
 	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {
+		return errors.Wrap(err, "schedule HTTP server with probes")
+	}
+
 	confContentYaml, err := objStoreConfig.Content()
 	if err != nil {
 		return err
@@ -302,12 +308,6 @@ func runReceive(
 		)
 	}
 
-	level.Debug(logger).Log("msg", "setting up http server")
-	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
-	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {
-		return errors.Wrap(err, "schedule HTTP server with probes")
-	}
-
 	level.Debug(logger).Log("msg", "setting up grpc server")
 	{
 		var (

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -483,6 +483,41 @@ func runRule(
 			close(cancel)
 		})
 	}
+	// Start query API + UI, metrics and status probe HTTP server.
+	level.Debug(logger).Log("msg", "setting up http server")
+	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	{
+		router := route.New()
+
+		// Redirect from / to /webRoutePrefix.
+		if webRoutePrefix != "" {
+			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
+				http.Redirect(w, r, webRoutePrefix, http.StatusFound)
+			})
+		}
+
+		router.WithPrefix(webRoutePrefix).Post("/-/reload", func(w http.ResponseWriter, r *http.Request) {
+			reload <- struct{}{}
+		})
+
+		flagsMap := map[string]string{
+			// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
+			"web.external-prefix": webExternalPrefix,
+			"web.prefix-header":   webPrefixHeaderName,
+		}
+
+		ins := extpromhttp.NewInstrumentationMiddleware(reg)
+
+		ui.NewRuleUI(logger, reg, ruleMgrs, alertQueryURL.String(), flagsMap).Register(router.WithPrefix(webRoutePrefix), ins)
+
+		api := v1.NewAPI(logger, reg, ruleMgrs)
+		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
+
+		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
+			return errors.Wrap(err, "schedule HTTP server with probes")
+		}
+	}
 	// Periodically update the addresses from static flags and file SD by resolving them using DNS SD if necessary.
 	{
 		ctx, cancel := context.WithCancel(context.Background())
@@ -495,7 +530,6 @@ func runRule(
 			cancel()
 		})
 	}
-	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	// Start gRPC server.
 	{
 		l, err := net.Listen("tcp", grpcBindAddr)
@@ -519,39 +553,6 @@ func runRule(
 			s.Stop()
 		})
 	}
-	// Start UI & metrics HTTP server.
-	{
-		router := route.New()
-
-		// Redirect from / to /webRoutePrefix.
-		if webRoutePrefix != "" {
-			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
-				http.Redirect(w, r, webRoutePrefix, http.StatusFound)
-			})
-		}
-
-		router.WithPrefix(webRoutePrefix).Post("/-/reload", func(w http.ResponseWriter, r *http.Request) {
-			reload <- struct{}{}
-		})
-
-		flagsMap := map[string]string{
-			// TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting.
-			"web.external-prefix": webExternalPrefix,
-			"web.prefix-header":   webPrefixHeaderName,
-		}
-
-		ins := extpromhttp.NewInstrumentationMiddleware(reg)
-
-		ui.NewRuleUI(logger, reg, ruleMgrs, alertQueryURL.String(), flagsMap).Register(router.WithPrefix(webRoutePrefix), ins)
-
-		api := v1.NewAPI(logger, reg, ruleMgrs)
-		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
-
-		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
-		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
-			return errors.Wrap(err, "schedule HTTP server with probes")
-		}
-	}
 
 	confContentYaml, err := objStoreConfig.Content()
 	if err != nil {

diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go
@@ -131,6 +131,7 @@ func runSidecar(
 		uploads = false
 	}
 
+	level.Debug(logger).Log("msg", "setting up http server")
 	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
 	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, comp); err != nil {

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -126,7 +126,12 @@ func runStore(
 	selectorRelabelConf *extflag.PathOrContent,
 	advertiseCompatibilityLabel bool,
 ) error {
+	level.Debug(logger).Log("msg", "setting up http server")
+	// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
 	statusProber := prober.NewProber(component, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil {
+		return errors.Wrap(err, "schedule HTTP server")
+	}
 
 	confContentYaml, err := objStoreConfig.Content()
 	if err != nil {
@@ -228,10 +233,6 @@ func runStore(
 		s.Stop()
 	})
 
-	if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, nil, component); err != nil {
-		return errors.Wrap(err, "schedule HTTP server")
-	}
-
 	level.Info(logger).Log("msg", "starting store node")
 	return nil
 }