Add /-/ready and /-/healty

Also add the associated functions to one of the
most important sub-system of the pushgateway: metric store.

The http handler will currently mostly reflect the status of
the metric store, but there isn't much else that can break.

Since the current implementation of the disk metric store does
most of the init job in its constructor, /-/ready isn't super
useful yet.

handler/handler_test.go

@@ -47,6 +47,36 @@ func (m *MockMetricStore) Shutdown() error {
 	return nil
 }
 
+func (m *MockMetricStore) Healthy() (bool, error) {
+	return true, nil
+}
+
+func (m *MockMetricStore) Ready() (bool, error) {
+	return true, nil
+}
+
+func TestHealthyReady(t *testing.T) {
+	mms := MockMetricStore{}
+	req, err := http.NewRequest("GET", "http://example.org/", &bytes.Buffer{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	healthyHandler := HealthyHandler(&mms)
+	readyHandler := ReadyHandler(&mms)
+
+	w := httptest.NewRecorder()
+	healthyHandler(w, req, httprouter.Params{})
+	if expected, got := http.StatusOK, w.Code; expected != got {
+		t.Errorf("Wanted status code %v, got %v.", expected, got)
+	}
+
+	readyHandler(w, req, httprouter.Params{})
+	if expected, got := http.StatusOK, w.Code; expected != got {
+		t.Errorf("Wanted status code %v, got %v.", expected, got)
+	}
+}
+
 func TestPush(t *testing.T) {
 	mms := MockMetricStore{}
 	handler := Push(&mms, false)

handler/misc.go

@@ -0,0 +1,63 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/prometheus/pushgateway/storage"
+)
+
+func HealthyHandler(
+	ms storage.MetricStore,
+) func(http.ResponseWriter, *http.Request, httprouter.Params) {
+	handlerFunc := prometheus.InstrumentHandlerFunc("healthy",
+		func(w http.ResponseWriter, _ *http.Request) {
+			healthy, err := ms.Healthy()
+			if healthy {
+				io.WriteString(w, "OK")
+			} else {
+				fmt.Fprintf(w, "FAIL: %v", err)
+			}
+		},
+	)
+	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		handlerFunc(w, r)
+	}
+
+}
+
+func ReadyHandler(
+	ms storage.MetricStore,
+) func(http.ResponseWriter, *http.Request, httprouter.Params) {
+	handlerFunc := prometheus.InstrumentHandlerFunc("ready",
+		func(w http.ResponseWriter, _ *http.Request) {
+			ready, err := ms.Ready()
+			if ready {
+				io.WriteString(w, "OK")
+			} else {
+				fmt.Fprintf(w, "FAIL: %v", err)
+			}
+		},
+	)
+	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		handlerFunc(w, r)
+	}
+
+}

main.go

@@ -67,6 +67,9 @@ func main() {
 	// prometheus.EnableCollectChecks(true)
 
 	r := httprouter.New()
+	r.Handle("GET", "/-/healthy", handler.HealthyHandler(ms))
+	r.Handle("GET", "/-/ready", handler.ReadyHandler(ms))
+
 	r.Handler("GET", *metricsPath, prometheus.Handler())
 
 	// Handlers for pushing and deleting metrics.

storage/diskmetricstore.go

@@ -15,6 +15,7 @@ package storage
 
 import (
 	"encoding/gob"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -61,6 +62,8 @@ func NewDiskMetricStore(
 	persistenceFile string,
 	persistenceInterval time.Duration,
 ) *DiskMetricStore {
+	// TODO: Do that outside of the constructor to allow the HTTP server to
+	//  serve /-/healthy and /-/ready earlier.
 	dms := &DiskMetricStore{
 		writeQueue:      make(chan WriteRequest, writeQueueCapacity),
 		drain:           make(chan struct{}),
@@ -134,6 +137,24 @@ func (dms *DiskMetricStore) Shutdown() error {
 	return <-dms.done
 }
 
+func (dms *DiskMetricStore) Healthy() (bool, error) {
+	// By taking the lock we check that there is no deadlock.
+	dms.lock.Lock()
+	defer dms.lock.Unlock()
+
+	// A pushgateway that cannot be written to should not be
+	// considered as healthy.
+	if len(dms.writeQueue) == cap(dms.writeQueue) {
+		return false, fmt.Errorf("Write queue is full")
+	}
+
+	return true, nil
+}
+
+func (dms *DiskMetricStore) Ready() (bool, error) {
+	return dms.Healthy()
+}
+
 func (dms *DiskMetricStore) loop(persistenceInterval time.Duration) {
 	lastPersist := time.Now()
 	persistScheduled := false

storage/diskmetricstore_test.go

@@ -538,6 +538,14 @@ func TestNoPersistence(t *testing.T) {
 	if err := checkMetricFamilies(dms); err != nil {
 		t.Error(err)
 	}
+
+	if ready, err := dms.Ready(); ready != true || err != nil {
+		t.Error(err)
+	}
+
+	if healthy, err := dms.Healthy(); healthy != true || err != nil {
+		t.Error(err)
+	}
 }
 
 func checkMetricFamilies(dms *DiskMetricStore, expectedMFs ...*dto.MetricFamily) error {

storage/interface.go

@@ -56,6 +56,13 @@ type MetricStore interface {
 	// undefinded state). If nil is returned, the MetricStore cannot be
 	// "restarted" again, but it can still be used for read operations.
 	Shutdown() error
+	// Returns true, nil if the MetricStore is currently working as expected
+	// or false, Error if it is not.
+	Healthy() (bool, error)
+	// Returns true, nil if the MetricStore is ready to be used (all files
+	// are opened and checkpoints have been restored) or false, Error if it
+	// is not.
+	Ready() (bool, error)
 }
 
 // WriteRequest is a request to change the MetricStore, i.e. to process it, a