introduce otel tracing

Signed-off-by: Jörn Friedrich Dreyer <[email protected]>

introduce otlp tracing

Signed-off-by: Jörn Friedrich Dreyer <[email protected]>

changelog/unreleased/otlp-tracing.md

@@ -0,0 +1,5 @@
+Enhancement: add otlp tracing exporter
+
+We can now use `tracing_exporter=otlp` to send traces using the otlp exporter.
+
+https://github.com/cs3org/reva/pull/3496

cmd/revad/runtime/runtime.go

@@ -76,6 +76,7 @@ func RunWithOptions(mainConf map[string]interface{}, pidFile string, opts ...Opt
 type coreConf struct {
 	MaxCPUs            string `mapstructure:"max_cpus"`
 	TracingEnabled     bool   `mapstructure:"tracing_enabled"`
+	TracingExporter    string `mapstructure:"tracing_exporter"`
 	TracingEndpoint    string `mapstructure:"tracing_endpoint"`
 	TracingCollector   string `mapstructure:"tracing_collector"`
 	TracingServiceName string `mapstructure:"tracing_service_name"`
@@ -149,9 +150,9 @@ func initServers(mainConf map[string]interface{}, log *zerolog.Logger, tp trace.
 
 func initTracing(conf *coreConf) trace.TracerProvider {
 	if conf.TracingEnabled {
-		rtrace.InitDefaultTracerProvider(conf.TracingCollector, conf.TracingEndpoint)
+		rtrace.InitDefaultTracerProvider(conf.TracingExporter, conf.TracingCollector, conf.TracingEndpoint)
 	}
-	return rtrace.GetTracerProvider(conf.TracingEnabled, conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName)
+	return rtrace.GetTracerProvider(conf.TracingEnabled, conf.TracingExporter, conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName)
 }
 
 func initCPUCount(conf *coreConf, log *zerolog.Logger) {

go.mod

@@ -73,6 +73,8 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4
 	go.opentelemetry.io/otel v1.11.1
 	go.opentelemetry.io/otel/exporters/jaeger v1.11.1
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1
 	go.opentelemetry.io/otel/sdk v1.11.1
 	go.opentelemetry.io/otel/trace v1.11.1
 	golang.org/x/crypto v0.1.0
@@ -99,6 +101,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cloudflare/circl v1.2.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
@@ -131,6 +134,7 @@ require (
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
 	github.com/hashicorp/consul/api v1.15.2 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -193,6 +197,8 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.5 // indirect
 	go.mongodb.org/mongo-driver v1.10.3 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect

go.sum

@@ -207,6 +207,8 @@ github.com/c0rby/go-cs3apis v0.0.0-20230110100311-5b424f1baa35/go.mod h1:UXha4Tg
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
+github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/ceph/go-ceph v0.18.0 h1:4WM6yAq/iqBDaeeADDiPKLqKiP0iZ4fffdgCr1lnOL4=
 github.com/ceph/go-ceph v0.18.0/go.mod h1:cflETVTBNAQM6jdr7hpNHHFHKYiJiWWcAeRDrRx/1ng=
@@ -525,6 +527,9 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaW
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/consul/api v1.15.2 h1:3Q/pDqvJ7udgt/60QOOW/p/PeKioQN+ncYzzCdN2av0=
 github.com/hashicorp/consul/api v1.15.2/go.mod h1:v6nvB10borjOuIwNRZYPZiHKrTM/AyrGtd0WVVodKM8=
@@ -947,18 +952,26 @@ go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4=
 go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE=
 go.opentelemetry.io/otel/exporters/jaeger v1.11.1 h1:F9Io8lqWdGyIbY3/SOGki34LX/l+7OL0gXNxjqwcbuQ=
 go.opentelemetry.io/otel/exporters/jaeger v1.11.1/go.mod h1:lRa2w3bQ4R4QN6zYsDgy7tEezgoKEu7Ow2g35Y75+KI=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 h1:X2GndnMCsUPh6CiY2a+frAbNsXaPLbB0soHRYhAZ5Ig=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 h1:MEQNafcNCB0uQIti/oHgU7CZpUMYQ7qigBwMVKycHvc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 h1:LYyG/f1W/jzAix16jbksJfMQFpOH/Ma6T639pVPMgfI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1/go.mod h1:QrRRQiY3kzAoYPNLP0W/Ikg0gR6V3LMc+ODSxr7yyvg=
 go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs=
 go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys=
 go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ=
 go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
+go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
@@ -1508,6 +1521,7 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
+google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=

internal/http/services/owncloud/ocdav/propfind/propfind.go

@@ -53,6 +53,7 @@ import (
 	"github.com/rs/zerolog"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
 )
 
@@ -233,6 +234,9 @@ func (p *Handler) HandleSpacesPropfind(w http.ResponseWriter, r *http.Request, s
 
 	depth, err := net.ParseDepth(dh)
 	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, "Invalid Depth header value")
+		span.SetAttributes(semconv.HTTPStatusCodeKey.Int(http.StatusBadRequest))
 		sublog.Debug().Str("depth", dh).Msg(err.Error())
 		w.WriteHeader(http.StatusBadRequest)
 		m := fmt.Sprintf("Invalid Depth header value: %v", dh)

pkg/micro/ocdav/option.go

@@ -48,6 +48,7 @@ type Options struct {
 	GatewayClient   gateway.GatewayAPIClient
 
 	TracingEnabled   bool
+	TracingExporter  string
 	TracingCollector string
 	TracingEndpoint  string
 
@@ -205,11 +206,12 @@ func LockSystem(val ocdav.LockSystem) Option {
 }
 
 // Tracing enables tracing
-func Tracing(trEndpoint string, trCollector string) Option {
+func Tracing(exporter, endpoint, collector string) Option {
 	return func(o *Options) {
 		o.TracingEnabled = true
-		o.TracingEndpoint = trEndpoint
-		o.TracingCollector = trCollector
+		o.TracingExporter = exporter
+		o.TracingEndpoint = endpoint
+		o.TracingCollector = collector
 	}
 }
 

pkg/micro/ocdav/service.go

@@ -67,7 +67,7 @@ func Service(opts ...Option) (micro.Service, error) {
 		server.Version(sopts.config.VersionString),
 	)
 
-	tp := rtrace.GetTracerProvider(sopts.TracingEnabled, sopts.TracingCollector, sopts.TracingEndpoint, sopts.Name)
+	tp := rtrace.GetTracerProvider(sopts.TracingEnabled, sopts.TracingExporter, sopts.TracingCollector, sopts.TracingEndpoint, sopts.Name)
 	revaService, err := ocdav.NewWith(&sopts.config, sopts.FavoriteManager, sopts.lockSystem, &sopts.Logger, tp, sopts.GatewayClient)
 	if err != nil {
 		return nil, err

pkg/trace/trace.go

@@ -26,11 +26,15 @@ import (
 	"strings"
 	"sync"
 
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -71,24 +75,39 @@ func ContextGetTracerProvider(ctx context.Context) trace.TracerProvider {
 }
 
 // InitDefaultTracerProvider initializes a global default TracerProvider at a package level.
-func InitDefaultTracerProvider(collectorEndpoint string, agentEndpoint string) {
+func InitDefaultTracerProvider(exporter, collector, endpoint string) {
 	defaultProvider.mutex.Lock()
 	defer defaultProvider.mutex.Unlock()
 	if !defaultProvider.initialized {
-		defaultProvider.provider = GetTracerProvider(true, collectorEndpoint, agentEndpoint, "reva default provider")
+		switch exporter {
+		case "otlp":
+			defaultProvider.provider = getOtlpTracerProvider(true, endpoint, "reva default otlp provider")
+		default:
+			defaultProvider.provider = getJaegerTracerProvider(true, collector, endpoint, "reva default jaeger provider")
+		}
 	}
 	defaultProvider.initialized = true
 }
 
+// GetTracerProvider returns a new TracerProvider, configure for the specified service
+func GetTracerProvider(enabled bool, exporter, collector, endpoint, serviceName string) trace.TracerProvider {
+	switch exporter {
+	case "otlp":
+		return getOtlpTracerProvider(enabled, endpoint, serviceName)
+	default:
+		return getJaegerTracerProvider(enabled, collector, endpoint, serviceName)
+	}
+}
+
 // DefaultProvider returns the "global" default TracerProvider
 func DefaultProvider() trace.TracerProvider {
 	defaultProvider.mutex.RLock()
 	defer defaultProvider.mutex.RUnlock()
 	return defaultProvider.provider
 }
 
-// GetTracerProvider returns a new TracerProvider, configure for the specified service
-func GetTracerProvider(enabled bool, collectorEndpoint string, agentEndpoint, serviceName string) trace.TracerProvider {
+// getJaegerTracerProvider returns a new TracerProvider, configure for the specified service
+func getJaegerTracerProvider(enabled bool, collector, endpoint, serviceName string) trace.TracerProvider {
 	if !enabled {
 		return trace.NewNoopTracerProvider()
 	}
@@ -101,11 +120,11 @@ func GetTracerProvider(enabled bool, collectorEndpoint string, agentEndpoint, se
 	var exp *jaeger.Exporter
 	var err error
 
-	if agentEndpoint != "" {
+	if endpoint != "" {
 		var agentHost string
 		var agentPort string
 
-		agentHost, agentPort, err = parseAgentConfig(agentEndpoint)
+		agentHost, agentPort, err = parseAgentConfig(endpoint)
 		if err != nil {
 			panic(err)
 		}
@@ -121,8 +140,8 @@ func GetTracerProvider(enabled bool, collectorEndpoint string, agentEndpoint, se
 		}
 	}
 
-	if collectorEndpoint != "" {
-		exp, err = jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collectorEndpoint)))
+	if collector != "" {
+		exp, err = jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(collector)))
 		if err != nil {
 			panic(err)
 		}
@@ -166,3 +185,48 @@ func parseAgentConfig(ae string) (string, string, error) {
 	}
 	return p[0], p[1], nil
 }
+
+// getOtelTracerProvider returns a new TracerProvider, configure for the specified service
+func getOtlpTracerProvider(enabled bool, endpoint string, serviceName string) trace.TracerProvider {
+	if !enabled {
+		return trace.NewNoopTracerProvider()
+	}
+
+	// default to 'reva' as service name if not set
+	if serviceName == "" {
+		serviceName = "reva"
+	}
+
+	//secureOption := otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))
+	//if len(insecure) > 0 {
+	secureOption := otlptracegrpc.WithInsecure()
+	//}
+
+	exporter, err := otlptrace.New(
+		context.Background(),
+		otlptracegrpc.NewClient(
+			secureOption,
+			otlptracegrpc.WithEndpoint(endpoint),
+		),
+	)
+
+	if err != nil {
+		panic(err)
+	}
+	resources, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			attribute.String("service.name", serviceName),
+			attribute.String("library.language", "go"),
+		),
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+		sdktrace.WithBatcher(exporter),
+		sdktrace.WithResource(resources),
+	)
+}