observability: add head(object) counter, latency, and error-count

* counting only remote heads for now

Signed-off-by: Alex Aizman <[email protected]>

ais/backend/common.go

@@ -119,6 +119,29 @@ func (b *base) init(snode *meta.Snode, tr stats.Tracker) {
 		},
 	)
 
+	// HEAD
+	b.metrics[stats.HeadCount] = prefix + "." + stats.HeadCount
+	b.metrics[stats.HeadLatencyTotal] = prefix + "." + stats.HeadLatencyTotal
+
+	tr.RegExtMetric(snode,
+		b.metrics[stats.HeadCount],
+		stats.KindCounter,
+		&stats.Extra{
+			Help:    "HEAD: total number of executed remote requests to a given backend",
+			StrName: "remote_head_count",
+			Labels:  labels,
+		},
+	)
+	tr.RegExtMetric(snode,
+		b.metrics[stats.HeadLatencyTotal],
+		stats.KindTotal,
+		&stats.Extra{
+			Help:    "HEAD: total cumulative time (nanoseconds) to execute remote requests",
+			StrName: "remote_head_ns_total",
+			Labels:  labels,
+		},
+	)
+
 	// version changed out-of-band
 	b.metrics[stats.VerChangeCount] = prefix + "." + stats.VerChangeCount
 	b.metrics[stats.VerChangeSize] = prefix + "." + stats.VerChangeSize

ais/target.go

@@ -1104,7 +1104,7 @@ func (t *target) objHead(r *http.Request, whdr http.Header, query url.Values, bc
 	} else {
 		// cold HEAD
 		var oa *cmn.ObjAttrs
-		oa, ecode, err = t.Backend(lom.Bck()).HeadObj(context.Background(), lom, r)
+		oa, ecode, err = t.HeadCold(lom, r)
 		if err != nil {
 			if ecode != http.StatusNotFound {
 				err = cmn.NewErrFailedTo(t, "HEAD", lom.Cname(), err)

ais/tgtimpl.go

@@ -211,6 +211,23 @@ func (t *target) GetColdBlob(params *core.BlobParams, oa *cmn.ObjAttrs) (xctn co
 	return xctn, err
 }
 
+func (t *target) HeadCold(lom *core.LOM, origReq *http.Request) (oa *cmn.ObjAttrs, ecode int, err error) {
+	var (
+		backend = t.Backend(lom.Bck())
+		now     = mono.NanoTime()
+	)
+	oa, ecode, err = backend.HeadObj(context.Background(), lom, origReq)
+	if err != nil {
+		t.statsT.IncErr(stats.ErrHeadCount)
+	} else {
+		t.statsT.AddMany(
+			cos.NamedVal64{Name: backend.MetricName(stats.HeadCount), Value: 1},
+			cos.NamedVal64{Name: backend.MetricName(stats.HeadLatencyTotal), Value: mono.SinceNano(now)},
+		)
+	}
+	return oa, ecode, err
+}
+
 func (t *target) Promote(params *core.PromoteParams) (ecode int, err error) {
 	lom := core.AllocLOM(params.ObjName)
 	if err = lom.InitBck(params.Bck.Bucket()); err == nil {

ais/tgts3.go

@@ -5,7 +5,6 @@
 package ais
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -316,7 +315,7 @@ func (t *target) headObjS3(w http.ResponseWriter, r *http.Request, items []strin
 		op.ObjAttrs = *lom.ObjAttrs()
 	} else {
 		// cold HEAD
-		objAttrs, ecode, err := t.Backend(lom.Bck()).HeadObj(context.Background(), lom, r)
+		objAttrs, ecode, err := t.HeadCold(lom, r)
 		if err != nil {
 			s3.WriteErr(w, r, err, ecode)
 			return

core/ldp.go

@@ -126,7 +126,7 @@ func (lom *LOM) CheckRemoteMD(locked, sync bool, origReq *http.Request) (res CRM
 		return CRMD{Eq: true}
 	}
 
-	oa, ecode, err := T.Backend(bck).HeadObj(context.Background(), lom, origReq)
+	oa, ecode, err := T.HeadCold(lom, origReq)
 	if err == nil {
 		if !lom.IsFeatureSet(feat.DisableColdGET) || lom.Equal(oa) {
 			debug.Assert(ecode == 0, ecode)

core/mock/target_mock.go

@@ -72,6 +72,10 @@ func (*TargetMock) GetCold(context.Context, *core.LOM, cmn.OWT) (int, error) {
 	return http.StatusOK, nil
 }
 
+func (*TargetMock) HeadCold(*core.LOM, *http.Request) (*cmn.ObjAttrs, int, error) {
+	return nil, 0, nil
+}
+
 func (*TargetMock) GetColdBlob(*core.BlobParams, *cmn.ObjAttrs) (core.Xact, error) {
 	return nil, nil
 }

core/target.go

@@ -122,6 +122,8 @@ type (
 
 		GetCold(ctx context.Context, lom *LOM, owt cmn.OWT) (ecode int, err error)
 
+		HeadCold(lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)
+
 		CopyObject(lom *LOM, dm DM, coi *CopyParams) (int64, error)
 		Promote(params *PromoteParams) (ecode int, err error)
 		HeadObjT2T(lom *LOM, si *meta.Snode) bool

stats/api.go

@@ -132,6 +132,8 @@ func LatencyToCounter(latency string) string {
 		return GetCount
 	case PutLatency, PutRedirLatency:
 		return PutCount
+	case HeadLatency:
+		return HeadCount
 	case ListLatency:
 		return ListCount
 	case AppendLatency:

stats/common.go

@@ -68,17 +68,19 @@ const (
 	// KindCounter:
 	// all basic counters are accompanied by the corresponding (errPrefix + kind) error count:
 	// e.g.: "get.n" => "err.get.n", "put.n" => "err.put.n", etc.
-	GetCount    = "get.n"    // GET(object) count = (cold + warm)
-	PutCount    = "put.n"    // ditto PUT
-	AppendCount = "append.n" // ditto etc.
-	DeleteCount = "del.n"    // ditto
-	RenameCount = "ren.n"    // ditto
-	ListCount   = "lst.n"    // list-objects
+	GetCount    = "get.n" // GET(object) count = (cold + warm)
+	PutCount    = "put.n" // ditto PUT
+	HeadCount   = "head.n"
+	AppendCount = "append.n"
+	DeleteCount = "del.n"
+	RenameCount = "ren.n"
+	ListCount   = "lst.n" // list-objects
 
 	// error counters
 	// see also: `IncErr`, `regCommon`, `ioErrNames`
 	ErrGetCount    = errPrefix + GetCount
 	ErrPutCount    = errPrefix + PutCount
+	ErrHeadCount   = errPrefix + HeadCount
 	ErrAppendCount = errPrefix + AppendCount
 	ErrDeleteCount = errPrefix + DeleteCount
 	ErrRenameCount = errPrefix + RenameCount
@@ -169,6 +171,11 @@ func (r *runner) regCommon(snode *meta.Snode) {
 			Help: "total number of executed PUT(object) requests",
 		},
 	)
+	r.reg(snode, HeadCount, KindCounter,
+		&Extra{
+			Help: "total number of executed HEAD(object) requests", // NOTE: currently, we only count remote ("cold") HEAD
+		},
+	)
 	r.reg(snode, AppendCount, KindCounter,
 		&Extra{
 			Help: "total number of executed APPEND(object) requests",
@@ -201,6 +208,11 @@ func (r *runner) regCommon(snode *meta.Snode) {
 			Help: "total number of PUT(object) errors",
 		},
 	)
+	r.reg(snode, ErrHeadCount, KindCounter,
+		&Extra{
+			Help: "total number of HEAD(object) errors", // ditto (HeadCount above)
+		},
+	)
 	r.reg(snode, ErrAppendCount, KindCounter,
 		&Extra{
 			Help: "total number of APPEND(object) errors",

stats/target_stats.go

@@ -65,6 +65,8 @@ const (
 	GetRedirLatency    = "get.redir.ns"
 	PutRedirLatency    = "put.redir.ns"
 	DownloadLatency    = "dl.ns"
+	HeadLatency        = "head.ns"
+	HeadLatencyTotal   = "head.ns.total"
 
 	// Dsort
 	DsortCreationReqCount    = "dsort.creation.req.n"
@@ -285,6 +287,16 @@ func (r *Trunner) RegMetrics(snode *meta.Snode) {
 			Help: "PUT: total cumulative time (nanoseconds)",
 		},
 	)
+	r.reg(snode, HeadLatency, KindLatency,
+		&Extra{
+			Help: "HEAD: average time (milliseconds) over the last periodic.stats_time interval",
+		},
+	)
+	r.reg(snode, HeadLatencyTotal, KindTotal,
+		&Extra{
+			Help: "HEAD: total cumulative time (nanoseconds)",
+		},
+	)
 	r.reg(snode, AppendLatency, KindLatency,
 		&Extra{
 			Help: "APPEND(object): average time (milliseconds) over the last periodic.stats_time interval",

xact/xs/blob_download.go

@@ -104,7 +104,7 @@ func RenewBlobDl(xid string, params *core.BlobParams, oa *cmn.ObjAttrs) xreg.Ren
 	if oa == nil {
 		// backend.HeadObj(), unless already done via prior (e.g. latest-ver or prefetch-threshold) check
 		// (in the latter case, oa.Size must be present)
-		oah, ecode, err := core.T.Backend(lom.Bck()).HeadObj(context.Background(), lom, nil /*origReq*/)
+		oah, ecode, err := core.T.HeadCold(lom, nil /*origReq*/)
 		if err != nil {
 			return xreg.RenewRes{Err: err}
 		}

xact/xs/prune.go

@@ -6,7 +6,6 @@
 package xs
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"time"
@@ -131,7 +130,7 @@ func (rp *prune) do(dst *core.LOM, _ []byte) error {
 			}
 		}
 	} else {
-		_, ecode, err = core.T.Backend(src.Bck()).HeadObj(context.Background(), src, nil /*origReq*/)
+		_, ecode, err = core.T.HeadCold(src, nil /*origReq*/)
 	}
 
 	if (err == nil && ecode == 0) || !cos.IsNotExist(err, ecode) /*not complaining*/ {