Loki: Log a crude lag metric for how far behind a client is. (#3236)

* Log the difference in time between the most recently received log entry and the current time as a crude measure of how far behind a promtail instance is.

* fix lint

pkg/distributor/http.go

@@ -3,6 +3,7 @@ package distributor
 import (
 	"math"
 	"net/http"
+	"time"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/dustin/go-humanize"
@@ -72,11 +73,16 @@ func ParseRequest(r *http.Request) (*logproto.PushRequest, error) {
 			totalEntries     int64
 		)
 
+		mostRecentEntry := time.Unix(0, 0)
+
 		for _, s := range req.Streams {
 			streamLabelsSize += int64(len(s.Labels))
 			for _, e := range s.Entries {
 				totalEntries++
 				entriesSize += int64(len(e.Line))
+				if e.Timestamp.After(mostRecentEntry) {
+					mostRecentEntry = e.Timestamp
+				}
 			}
 		}
 
@@ -96,6 +102,7 @@ func ParseRequest(r *http.Request) (*logproto.PushRequest, error) {
 			"streamLabelsSize", humanize.Bytes(uint64(streamLabelsSize)),
 			"entriesSize", humanize.Bytes(uint64(entriesSize)),
 			"totalSize", humanize.Bytes(uint64(entriesSize+streamLabelsSize)),
+			"mostRecentLagMs", time.Since(mostRecentEntry).Milliseconds(),
 		)
 	}()