Merge pull request #514 from tobert/add-otel-plumbing

feat: add otel plumbing & grpc instrumentation

README.md

@@ -68,6 +68,30 @@ The most interesting targets are `make all` (or just `make`) and `make images`.
 `make all` builds all the binaries for your host OS and CPU to enable running directly.
 `make images` will build all the binaries for Linux/x86_64 and build docker images with them.
 
+## Configuring OpenTelemetry
+
+Rather than adding a bunch of command line options or a config file, OpenTelemetry
+is configured via environment variables. The most relevant ones are below, for others
+see https://github.com/equinix-labs/otel-init-go
+
+Currently this is just for tracing, metrics needs to be discussed with the community.
+
+| Env Variable                  | Required | Default   |
+| ----------------------------- | -------- | --------- |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | n        | localhost |
+| `OTEL_EXPORTER_OTLP_INSECURE` | n        | false     |
+| `OTEL_LOG_LEVEL`              | n        | info      |
+
+To work with a local [opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector),
+try the following. For examples of how to set up the collector to relay to various services
+take a look at [otel-cli](https://github.com/packethost/otel-cli)
+
+```
+export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317
+export OTEL_EXPORTER_OTLP_INSECURE=true
+./cmd/tink-server/tink-server <stuff>
+```
+
 ## Website
 
 For complete documentation, please visit the Tinkerbell project hosted at [tinkerbell.org](https://tinkerbell.org).

client/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/tinkerbell/tink/protos/hardware"
 	"github.com/tinkerbell/tink/protos/template"
 	"github.com/tinkerbell/tink/protos/workflow"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 )
@@ -125,7 +126,11 @@ func GetConnection() (*grpc.ClientConn, error) {
 		return nil, errors.New("undefined TINKERBELL_GRPC_AUTHORITY")
 	}
 	creds := credentials.NewClientTLSFromCert(cp, "")
-	conn, err := grpc.Dial(grpcAuthority, grpc.WithTransportCredentials(creds))
+	conn, err := grpc.Dial(grpcAuthority,
+		grpc.WithTransportCredentials(creds),
+		grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
+		grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
+	)
 	if err != nil {
 		return nil, errors.Wrap(err, "connect to tinkerbell server")
 	}

cmd/tink-cli/main.go

@@ -1,16 +1,21 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 
+	"github.com/equinix-labs/otel-init-go/otelinit"
 	"github.com/tinkerbell/tink/cmd/tink-cli/cmd"
 )
 
 // version is set at build time
 var version = "devel"
 
 func main() {
+	ctx, otelShutdown := otelinit.InitOpenTelemetry(context.Background(), "github.com/tinkerbell/tink")
+	defer otelShutdown(ctx)
+
 	if err := cmd.Execute(version); err != nil {
 		fmt.Fprintln(os.Stderr, err.Error())
 		os.Exit(1)

cmd/tink-server/main.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/equinix-labs/otel-init-go/otelinit"
 	"github.com/packethost/pkg/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -102,10 +103,14 @@ func main() {
 	}
 	defer logger.Close()
 
+	ctx := context.Background()
+	ctx, otelShutdown := otelinit.InitOpenTelemetry(ctx, "github.com/tinkerbell/tink")
+	defer otelShutdown(ctx)
+
 	config := &DaemonConfig{}
 
 	cmd := NewRootCommand(config, logger)
-	if err := cmd.ExecuteContext(context.Background()); err != nil {
+	if err := cmd.ExecuteContext(ctx); err != nil {
 		os.Exit(1)
 	}
 

cmd/tink-worker/internal/action.go

@@ -52,7 +52,7 @@ func (w *Worker) createContainer(ctx context.Context, cmd []string, wfID string,
 
 	hostConfig.Binds = append(hostConfig.Binds, action.GetVolumes()...)
 	w.logger.With("command", cmd).Info("creating container")
-	resp, err := w.registryClient.ContainerCreate(ctx, config, hostConfig, nil, action.GetName())
+	resp, err := w.registryClient.ContainerCreate(ctx, config, hostConfig, nil, nil, action.GetName())
 	if err != nil {
 		return "", errors.Wrap(err, "DOCKER CREATE")
 	}

cmd/tink-worker/main.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"context"
 	"os"
 
+	"github.com/equinix-labs/otel-init-go/otelinit"
 	"github.com/packethost/pkg/log"
 	"github.com/tinkerbell/tink/cmd/tink-worker/cmd"
 )
@@ -21,9 +23,11 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-
 	defer logger.Close()
 
+	ctx, otelShutdown := otelinit.InitOpenTelemetry(context.Background(), "github.com/tinkerbell/tink")
+	defer otelShutdown(ctx)
+
 	rootCmd := cmd.NewRootCommand(version, logger)
 
 	if err := rootCmd.Execute(); err != nil {

go.mod

@@ -3,43 +3,36 @@ module github.com/tinkerbell/tink
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
-	github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/docker v20.10.7+incompatible
+	github.com/equinix-labs/otel-init-go v0.0.1
 	github.com/go-openapi/strfmt v0.19.3 // indirect
-	github.com/golang/protobuf v1.5.0
-	github.com/google/go-cmp v0.5.5
-	github.com/google/uuid v1.1.2
+	github.com/golang/protobuf v1.5.2
+	github.com/google/go-cmp v0.5.6
+	github.com/google/uuid v1.2.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/grpc-ecosystem/grpc-gateway v1.15.2
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481
-	github.com/mattn/go-runewidth v0.0.5 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.3.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.7.0
+	github.com/spf13/viper v1.8.1
 	github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816
-	github.com/stretchr/testify v1.6.1
-	github.com/testcontainers/testcontainers-go v0.9.0
+	github.com/stretchr/testify v1.7.0
+	github.com/testcontainers/testcontainers-go v0.11.1
 	go.mongodb.org/mongo-driver v1.1.2 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.16.0 // indirect
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
-	golang.org/x/tools v0.1.0 // indirect
-	google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877
-	google.golang.org/grpc v1.32.0
-	google.golang.org/protobuf v1.26.0
-	gopkg.in/yaml.v2 v2.3.0
-	gotest.tools v2.2.0+incompatible // indirect
-	honnef.co/go/tools v0.0.1-2020.1.4 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.22.0
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
+	google.golang.org/grpc v1.39.0
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 replace github.com/stormcat24/protodep => github.com/ackintosh/protodep v0.0.0-20200728152107-abf8eb579d6c