From ae8a2d3b9b3d8e31936d388ce609af2079a184c3 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Thu, 6 Feb 2025 14:26:53 +0100 Subject: [PATCH] chore: Print trace ID in panic when available. --- pkg/util/server/recovery.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/util/server/recovery.go b/pkg/util/server/recovery.go index 58eac396dd750..d321e5ac04125 100644 --- a/pkg/util/server/recovery.go +++ b/pkg/util/server/recovery.go @@ -10,8 +10,10 @@ import ( "github.com/grafana/dskit/httpgrpc" "github.com/grafana/dskit/middleware" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" + "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/uber/jaeger-client-go" "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/v3/pkg/util/constants" @@ -30,20 +32,20 @@ var ( return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer func() { if p := recover(); p != nil { - WriteError(onPanic(p), w) + WriteError(onPanic(req.Context(), p), w) } }() next.ServeHTTP(w, req) }) }) - RecoveryGRPCStreamInterceptor = grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(onPanic)) - RecoveryGRPCUnaryInterceptor = grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(onPanic)) + RecoveryGRPCStreamInterceptor = grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(onPanic)) + RecoveryGRPCUnaryInterceptor = grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(onPanic)) RecoveryMiddleware queryrangebase.Middleware = queryrangebase.MiddlewareFunc(func(next queryrangebase.Handler) queryrangebase.Handler { return queryrangebase.HandlerFunc(func(ctx context.Context, req queryrangebase.Request) (res queryrangebase.Response, err error) { defer func() { if p := recover(); p != nil { - err = onPanic(p) + err = onPanic(ctx, p) } }() res, err = next.Do(ctx, req) @@ -52,11 +54,27 @@ var ( }) ) -func onPanic(p interface{}) error { +func onPanic(ctx context.Context, p interface{}) error { stack := make([]byte, maxStacksize) stack = stack[:runtime.Stack(stack, true)] + traceID := jaegerTraceID(ctx) + // keep a multiline stack - fmt.Fprintf(os.Stderr, "panic: %v\n%s", p, stack) + fmt.Fprintf(os.Stderr, "panic: %v %s\n%s", p, traceID, stack) panicTotal.Inc() return httpgrpc.Errorf(http.StatusInternalServerError, "error while processing request: %v", p) } + +func jaegerTraceID(ctx context.Context) string { + span := opentracing.SpanFromContext(ctx) + if span == nil { + return "" + } + + spanContext, ok := span.Context().(jaeger.SpanContext) + if !ok { + return "" + } + + return "traceID=" + spanContext.TraceID().String() +}