From befb0257e8a3d49cb80b00794a4892abfb7ccb72 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Thu, 4 Mar 2021 19:35:12 +0530 Subject: [PATCH] Upstream changes for PR #3361 - Added request logging for gRPC and HTTP in the server side (#3862) * Added request logging for grpc Signed-off-by: Yash Sharma Added request logging for grpc Signed-off-by: Yash Sharma changed the logging option to log start and finish of a call Signed-off-by: Yash Sharma added flags for enabling/disabling request logging for grpc Signed-off-by: Yash Sharma nitpicks due to make docs Signed-off-by: Yash Sharma added a changelog Signed-off-by: Yash Sharma nitpicks Signed-off-by: Yash Sharma configure option for logging Signed-off-by: Yash Sharma changelog nitpick Signed-off-by: Yash Sharma added changelog Signed-off-by: Yash Sharma more nitpicks Signed-off-by: Yash Sharma renamed requestLoggingDecision to reqLogDecision and some nitcpicks Signed-off-by: Yash Sharma Added the check of reusing the request-id if present Signed-off-by: Yash Sharma make docs nitpick Signed-off-by: Yash Sharma Added the default level of debug and error codes for logging Signed-off-by: Yash Sharma Add the levels for logging Signed-off-by: Yash Sharma Added a YAML flag for request logging config Signed-off-by: Yash Sharma Changed the setup signature to accept a reqlog param to be shared among all the components of Thanos. Changed the signature of decision for logging, and pushed the new yaml config among all the middleware Signed-off-by: Yash Sharma Updated the grpc middleware package to include the latest signature of the decider Signed-off-by: Yash Sharma Added new config files for YAML parsing Signed-off-by: Yash Sharma changelog nitpick Signed-off-by: Yash Sharma linting nitpicks Signed-off-by: Yash Sharma Added deprecation warning and added a function to reroute logging through one of the flags Signed-off-by: Yash Sharma docs nitpicks Signed-off-by: Yash Sharma bug fix for reverse logging condition Signed-off-by: Yash Sharma fixed some bugs for evaluating the options and added a warning for deprecating flag of log.request.decision Signed-off-by: Yash Sharma removed gophercloud from go.mod Signed-off-by: Yash Sharma self addressed comments Signed-off-by: Yash Sharma * Added a dummy variable which was used by the, to be deprecated flag of request logging * Added a warning that request logging is disabled for tools * Removed a flag which is to be deprecated in the next release Signed-off-by: Yash Sharma * Added a fail-first approach if the request logging is incorrectly enabled. Signed-off-by: Yash Sharma Renaming of functions Signed-off-by: Yash Sharma Removed indentation and simplified else conditions Signed-off-by: Yash Sharma break down the yaml struct for grpc and http into its individual configs Signed-off-by: Yash Sharma make docs nits Signed-off-by: Yash Sharma modify changelog Signed-off-by: Yash Sharma modified the signature of setup function to the original one Signed-off-by: Yash Sharma removed the message for the flags Signed-off-by: Yash Sharma make docs nits Signed-off-by: Yash Sharma Added auto generation scripts for req logging Signed-off-by: Yash Sharma removed request logging from compactor Signed-off-by: Yash Sharma remove verbose warn messages Signed-off-by: Yash Sharma changed pass by value to pass by reference Signed-off-by: Yash Sharma removed occurence of os.Exit Signed-off-by: Yash Sharma make docs nits Signed-off-by: Yash Sharma revert compact.go to master Signed-off-by: Yash Sharma rename ReqlogConfig to RequestConfig Signed-off-by: Yash Sharma Added a validation check so that all the configs are checked before a component starts Signed-off-by: Yash Sharma nits Signed-off-by: Yash Sharma Modify the message for request.logging flag for having a default value as null Signed-off-by: Yash Sharma remove a line from the flags Signed-off-by: Yash Sharma remove a deceptive comment Signed-off-by: Yash Sharma changed the var name to small caps Signed-off-by: Yash Sharma change errors.Errorf to Wrapf for providing context to error message Signed-off-by: Yash Sharma move changelog entry to unreleased tag Signed-off-by: Yash Sharma changed request.logging to `request.logging` in the flags Signed-off-by: Yash Sharma * added changelog entry Signed-off-by: Yash Sharma * changed changelog Signed-off-by: Yash Sharma * renamed decidehttpflag and decidegrpcflag Signed-off-by: Yash Sharma * addressed reviewers comments Signed-off-by: Yash Sharma --- CHANGELOG.md | 1 + cmd/thanos/compact.go | 2 +- cmd/thanos/query.go | 30 +++-- cmd/thanos/query_frontend.go | 16 ++- cmd/thanos/receive.go | 15 ++- cmd/thanos/rule.go | 31 +++-- cmd/thanos/sidecar.go | 16 ++- cmd/thanos/store.go | 26 +++- cmd/thanos/tools_bucket.go | 2 +- docs/components/query-frontend.md | 26 +++- docs/components/query.md | 26 +++- docs/components/receive.md | 10 ++ docs/components/rule.md | 26 +++- docs/components/sidecar.md | 10 ++ docs/components/store.md | 10 ++ go.mod | 3 +- go.sum | 5 + pkg/extgrpc/client.go | 2 +- pkg/extkingpin/flags.go | 11 ++ pkg/logging/grpc.go | 207 ++++++++++++++++++++++++++++++ pkg/logging/http.go | 123 +++++++++++++++++- pkg/logging/options.go | 138 +++++++++++++++++++- pkg/logging/yaml_parser.go | 40 ++++++ pkg/server/grpc/grpc.go | 13 +- pkg/store/multitsdb.go | 2 +- pkg/store/proxy.go | 2 +- pkg/tracing/grpc.go | 4 +- scripts/cfggen/main.go | 8 ++ 28 files changed, 733 insertions(+), 72 deletions(-) create mode 100644 pkg/logging/grpc.go create mode 100644 pkg/logging/yaml_parser.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 43eeea7ef5..d6a090d96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re ### Added +- [#3862](https://github.com/thanos-io/thanos/pull/3862) Sidecar, Store, Query, Ruler, Receiver, Query-Frontend: Added request logging for gRPC and HTTP in the server side. - [#3740](https://github.com/thanos-io/thanos/pull/3740) Query: Added `--query.default-step` flag to set default step. - [#3700](https://github.com/thanos-io/thanos/pull/3700) ui: make old bucket viewer UI work with vanilla Prometheus blocks - [#2641](https://github.com/thanos-io/thanos/issues/2641) Query Frontend: Added `--query-range.request-downsampled` flag enabling additional queries for downsampled data in case of empty or incomplete response to range request. diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 0e87870671..5587b578e2 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -464,7 +464,7 @@ func runCompact( global.Register(r, false, ins) // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { + opts := []logging.Option{logging.WithDecider(func(_ string, _ error) logging.Decision { return logging.NoLogCall })} logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 3bed633de3..10f18cc83d 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -25,6 +26,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" v1 "github.com/thanos-io/thanos/pkg/api/query" "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" @@ -66,7 +68,7 @@ func registerQuery(app *extkingpin.App) { webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the UI query web interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String() - requestLoggingDecision := cmd.Flag("log.request.decision", "Request Logging for logging the start and end of requests. LogFinishCall is enabled by default. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("LogFinishCall").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall") + reqLogDecision := cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall: Logs the finish call of the requests. LogStartAndFinishCall: Logs the start and finish call of the requests. NoLogCall: Disable request logging.").Default("").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") queryTimeout := extkingpin.ModelDuration(cmd.Flag("query.timeout", "Maximum time to process query by query node."). Default("2m")) @@ -136,6 +138,7 @@ func registerQuery(app *extkingpin.App) { Default("1s")) storeResponseTimeout := extkingpin.ModelDuration(cmd.Flag("store.response-timeout", "If a Store doesn't send any data in this specified duration then a Store will be ignored and partial data will be returned if it's enabled. 0 disables timeout.").Default("0ms")) + reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { selectorLset, err := parseFlagLabels(*selectorLabels) @@ -155,6 +158,16 @@ func registerQuery(app *extkingpin.App) { return errors.Errorf("Address %s is duplicated for --metadata flag.", dup) } + httpLogOpts, err := logging.ParseHTTPOptions(*reqLogDecision, reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + + tagOpts, grpcLogOpts, err := logging.ParsegRPCOptions(*reqLogDecision, reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + var fileSD *file.Discovery if len(*fileSDFiles) > 0 { conf := &file.SDConfig{ @@ -177,7 +190,9 @@ func registerQuery(app *extkingpin.App) { logger, reg, tracer, - *requestLoggingDecision, + httpLogOpts, + grpcLogOpts, + tagOpts, *grpcBindAddr, time.Duration(*grpcGracePeriod), *grpcCert, @@ -230,7 +245,9 @@ func runQuery( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - requestLoggingDecision string, + httpLogOpts []logging.Option, + grpcLogOpts []grpc_logging.Option, + tagOpts []tags.Option, grpcBindAddr string, grpcGracePeriod time.Duration, grpcCert string, @@ -471,10 +488,7 @@ func runQuery( } // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { - return logging.LogDecision[requestLoggingDecision] - })} - logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) + logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...) ins := extpromhttp.NewInstrumentationMiddleware(reg) // TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting. @@ -529,7 +543,7 @@ func runQuery( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, + s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(proxy)), grpcserver.WithServer(rules.RegisterRulesServer(rulesProxy)), grpcserver.WithServer(metadata.RegisterMetadataServer(metadataProxy)), diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 8f0a9c56fb..2fc9626120 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -125,10 +125,16 @@ func registerQueryFrontend(app *extkingpin.App) { "If multiple headers match the request, the first matching arg specified will take precedence. "+ "If no headers match 'anonymous' will be used.").PlaceHolder("").StringsVar(&cfg.orgIdHeaders) - cmd.Flag("log.request.decision", "Request Logging for logging the start and end of requests. LogFinishCall is enabled by default. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("LogFinishCall").EnumVar(&cfg.RequestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall") + cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("").EnumVar(&cfg.RequestLoggingDecision, "NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") + reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { - return runQueryFrontend(g, logger, reg, tracer, cfg, comp) + httpLogOpts, err := logging.ParseHTTPOptions(cfg.RequestLoggingDecision, reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + + return runQueryFrontend(g, logger, reg, tracer, httpLogOpts, cfg, comp) }) } @@ -137,6 +143,7 @@ func runQueryFrontend( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, + httpLogOpts []logging.Option, cfg *queryFrontendConfig, comp component.Component, ) error { @@ -201,10 +208,7 @@ func runQueryFrontend( ) // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { - return logging.LogDecision[cfg.RequestLoggingDecision] - })} - logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) + logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...) ins := extpromhttp.NewInstrumentationMiddleware(reg) // Start metrics HTTP server. diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 6d401a7d08..5a00e0ff79 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -14,6 +14,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -25,6 +27,7 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/extkingpin" + "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extflag" @@ -102,6 +105,8 @@ func registerReceive(app *extkingpin.App) { "about order."). Default("false").Hidden().Bool() + reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) + cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { @@ -112,6 +117,11 @@ func registerReceive(app *extkingpin.App) { return errors.New("no external labels configured for receive, uniquely identifying external labels must be configured (ideally with `receive_` prefix); see https://thanos.io/tip/thanos/storage.md#external-labels for details.") } + tagOpts, grpcLogOpts, err := logging.ParsegRPCOptions("", reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + tsdbOpts := &tsdb.Options{ MinBlockDuration: int64(time.Duration(*tsdbMinBlockDuration) / time.Millisecond), MaxBlockDuration: int64(time.Duration(*tsdbMaxBlockDuration) / time.Millisecond), @@ -138,6 +148,7 @@ func registerReceive(app *extkingpin.App) { logger, reg, tracer, + grpcLogOpts, tagOpts, *grpcBindAddr, time.Duration(*grpcGracePeriod), *grpcCert, @@ -180,6 +191,8 @@ func runReceive( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, + grpcLogOpts []grpc_logging.Option, + tagOpts []tags.Option, grpcBindAddr string, grpcGracePeriod time.Duration, grpcCert string, @@ -494,7 +507,7 @@ func runReceive( WriteableStoreServer: webHandler, } - s = grpcserver.New(logger, &receive.UnRegisterer{Registerer: reg}, tracer, comp, grpcProbe, + s = grpcserver.New(logger, &receive.UnRegisterer{Registerer: reg}, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(rw)), grpcserver.WithServer(store.RegisterWritableStoreServer(rw)), grpcserver.WithListen(grpcBindAddr), diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index f45396dabf..7f20a9cdd9 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -17,6 +17,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -99,7 +101,7 @@ func registerRule(app *extkingpin.App) { webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the UI query web interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String() - requestLoggingDecision := cmd.Flag("log.request.decision", "Request Logging for logging the start and end of requests. LogFinishCall is enabled by default. LogFinishCall : Logs the finish call of the requests. LogStartAndFinishCall : Logs the start and finish call of the requests. NoLogCall : Disable request logging.").Default("LogFinishCall").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall") + reqLogDecision := cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall: Logs the finish call of the requests. LogStartAndFinishCall: Logs the start and finish call of the requests. NoLogCall: Disable request logging.").Default("").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") objStoreConfig := extkingpin.RegisterCommonObjStoreFlags(cmd, "", false) @@ -132,6 +134,8 @@ func registerRule(app *extkingpin.App) { hashFunc := cmd.Flag("hash-func", "Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: \"\", \"SHA256\"."). Default("").Enum("SHA256", "") + reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) + cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, reload <-chan struct{}, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { @@ -180,11 +184,23 @@ func registerRule(app *extkingpin.App) { return errors.New("--alertmanagers.url and --alertmanagers.config* parameters cannot be defined at the same time") } + httpLogOpts, err := logging.ParseHTTPOptions(*reqLogDecision, reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + + tagOpts, grpcLogOpts, err := logging.ParsegRPCOptions(*reqLogDecision, reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + return runRule(g, logger, reg, tracer, - *requestLoggingDecision, + httpLogOpts, + grpcLogOpts, + tagOpts, reload, lset, *alertmgrs, @@ -275,7 +291,9 @@ func runRule( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, - requestLoggingDecision string, + httpLogOpts []logging.Option, + grpcLogOpts []grpc_logging.Option, + tagOpts []tags.Option, reloadSignal <-chan struct{}, lset labels.Labels, alertmgrURLs []string, @@ -561,7 +579,7 @@ func runRule( } // TODO: Add rules API implementation when ready. - s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, + s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(tsdbStore)), grpcserver.WithServer(thanosrules.RegisterRulesServer(ruleMgr)), grpcserver.WithListen(grpcBindAddr), @@ -603,10 +621,7 @@ func runRule( ins := extpromhttp.NewInstrumentationMiddleware(reg) // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { - return logging.LogDecision[requestLoggingDecision] - })} - logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) + logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...) // TODO(bplotka in PR #513 review): pass all flags, not only the flags needed by prefix rewriting. ui.NewRuleUI(logger, reg, ruleMgr, alertQueryURL.String(), webExternalPrefix, webPrefixHeaderName).Register(router, ins) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 90ecd89e82..dd93addc85 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -13,6 +13,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -27,6 +29,7 @@ import ( "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" thanoshttp "github.com/thanos-io/thanos/pkg/http" + "github.com/thanos-io/thanos/pkg/logging" meta "github.com/thanos-io/thanos/pkg/metadata" thanosmodel "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" @@ -48,6 +51,11 @@ func registerSidecar(app *extkingpin.App) { conf := &sidecarConfig{} conf.registerFlag(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, _ bool) error { + tagOpts, grpcLogOpts, err := logging.ParsegRPCOptions("", conf.reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + rl := reloader.New(log.With(logger, "component", "reloader"), extprom.WrapRegistererWithPrefix("thanos_sidecar_", reg), &reloader.Options{ @@ -59,7 +67,7 @@ func registerSidecar(app *extkingpin.App) { RetryInterval: conf.reloader.retryInterval, }) - return runSidecar(g, logger, reg, tracer, rl, component.Sidecar, *conf) + return runSidecar(g, logger, reg, tracer, rl, component.Sidecar, *conf, grpcLogOpts, tagOpts) }) } @@ -71,6 +79,8 @@ func runSidecar( reloader *reloader.Reloader, comp component.Component, conf sidecarConfig, + grpcLogOpts []grpc_logging.Option, + tagOpts []tags.Option, ) error { var m = &promMetadata{ promURL: conf.prometheus.url, @@ -216,7 +226,7 @@ func runSidecar( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, comp, grpcProbe, + s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(promStore)), grpcserver.WithServer(rules.RegisterRulesServer(rules.NewPrometheus(conf.prometheus.url, c, m.Labels))), grpcserver.WithServer(meta.RegisterMetadataServer(meta.NewPrometheus(conf.prometheus.url, c))), @@ -390,6 +400,7 @@ type sidecarConfig struct { connection connConfig tsdb tsdbConfig reloader reloaderConfig + reqLogConfig *extflag.PathOrContent objStore extflag.PathOrContent shipper shipperConfig limitMinTime thanosmodel.TimeOrDurationValue @@ -402,6 +413,7 @@ func (sc *sidecarConfig) registerFlag(cmd extkingpin.FlagClause) { sc.connection.registerFlag(cmd) sc.tsdb.registerFlag(cmd) sc.reloader.registerFlag(cmd) + sc.reqLogConfig = extkingpin.RegisterRequestLoggingFlags(cmd) sc.objStore = *extkingpin.RegisterCommonObjStoreFlags(cmd, "", false) sc.shipper.registerFlag(cmd) cmd.Flag("min-time", "Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened later than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y."). diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index d3f78348d3..0d1a2574b5 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -10,6 +10,8 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" "github.com/oklog/run" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -116,6 +118,7 @@ func registerStore(app *extkingpin.App) { webExternalPrefix := cmd.Flag("web.external-prefix", "Static prefix for all HTML links and redirect URLs in the bucket web UI interface. Actual endpoints are still served on / or the web.route-prefix. This allows thanos bucket web UI to be served behind a reverse proxy that strips a URL sub-path.").Default("").String() webPrefixHeaderName := cmd.Flag("web.prefix-header", "Name of HTTP request header used for dynamic prefixing of UI links and redirects. This option is ignored if web.external-prefix argument is set. Security risk: enable this option only if a reverse proxy in front of thanos is resetting the header. The --web.prefix-header=X-Forwarded-Prefix option can be useful, for example, if Thanos UI is served via Traefik reverse proxy with PathPrefixStrip option enabled, which sends the stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path.").Default("").String() + reqLogConfig := extkingpin.RegisterRequestLoggingFlags(cmd) cmd.Setup(func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ <-chan struct{}, debugLogging bool) error { if minTime.PrometheusTimestamp() > maxTime.PrometheusTimestamp() { @@ -123,10 +126,23 @@ func registerStore(app *extkingpin.App) { minTime, maxTime) } + httpLogOpts, err := logging.ParseHTTPOptions("", reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + + tagOpts, grpcLogOpts, err := logging.ParsegRPCOptions("", reqLogConfig) + if err != nil { + return errors.Wrap(err, "error while parsing config for request logging") + } + return runStore(g, logger, reg, tracer, + httpLogOpts, + grpcLogOpts, + tagOpts, indexCacheConfig, objStoreConfig, *dataDir, @@ -172,6 +188,9 @@ func runStore( logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, + httpLogOpts []logging.Option, + grpcLogOpts []grpc_logging.Option, + tagOpts []tags.Option, indexCacheConfig *extflag.PathOrContent, objStoreConfig *extflag.PathOrContent, dataDir string, @@ -366,7 +385,7 @@ func runStore( return errors.Wrap(err, "setup gRPC server") } - s := grpcserver.New(logger, reg, tracer, component, grpcProbe, + s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, component, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(bs)), grpcserver.WithListen(grpcBindAddr), grpcserver.WithGracePeriod(grpcGracePeriod), @@ -391,10 +410,7 @@ func runStore( compactorView.Register(r, true, ins) // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { - return logging.NoLogCall - })} - logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) + logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...) api := blocksAPI.NewBlocksAPI(logger, "", flagsMap) api.Register(r.WithPrefix("/api/v1"), tracer, logger, ins, logMiddleware) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 6c46597f24..6d612af23a 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -351,7 +351,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC api := v1.NewBlocksAPI(logger, *label, flagsMap) // Configure Request Logging for HTTP calls. - opts := []logging.Option{logging.WithDecider(func() logging.Decision { + opts := []logging.Option{logging.WithDecider(func(_ string, _ error) logging.Decision { return logging.NoLogCall })} logMiddleware := logging.NewHTTPServerMiddleware(logger, opts...) diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index a7ae37291e..75e32f54a5 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -235,12 +235,24 @@ Flags: headers match the request, the first matching arg specified will take precedence. If no headers match 'anonymous' will be used. - --log.request.decision=LogFinishCall - Request Logging for logging the start and end - of requests. LogFinishCall is enabled by - default. LogFinishCall : Logs the finish call - of the requests. LogStartAndFinishCall : Logs - the start and finish call of the requests. - NoLogCall : Disable request logging. + --log.request.decision= Deprecation Warning - This flag would be soon + deprecated, and replaced with + `request.logging-config`. Request Logging for + logging the start and end of requests. By + default this flag is disabled. LogFinishCall : + Logs the finish call of the requests. + LogStartAndFinishCall : Logs the start and + finish call of the requests. NoLogCall : + Disable request logging. + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 ``` diff --git a/docs/components/query.md b/docs/components/query.md index 657e8ae7aa..19f8067dd9 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -357,13 +357,15 @@ Flags: stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path. - --log.request.decision=LogFinishCall - Request Logging for logging the start and end - of requests. LogFinishCall is enabled by - default. LogFinishCall : Logs the finish call - of the requests. LogStartAndFinishCall : Logs - the start and finish call of the requests. - NoLogCall : Disable request logging. + --log.request.decision= Deprecation Warning - This flag would be soon + deprecated, and replaced with + `request.logging-config`. Request Logging for + logging the start and end of requests. By + default this flag is disabled. LogFinishCall: + Logs the finish call of the requests. + LogStartAndFinishCall: Logs the start and + finish call of the requests. NoLogCall: Disable + request logging. --query.timeout=2m Maximum time to process query by query node. --query.max-concurrent=20 Maximum number of queries processed concurrently by query node. @@ -440,5 +442,15 @@ Flags: specified duration then a Store will be ignored and partial data will be returned if it's enabled. 0 disables timeout. + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 ``` diff --git a/docs/components/receive.md b/docs/components/receive.md index 47a71febdc..bbc3e9e133 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -191,5 +191,15 @@ Flags: happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: "", "SHA256". + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 ``` diff --git a/docs/components/rule.md b/docs/components/rule.md index 6a3a862269..50bd576eb3 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -349,13 +349,15 @@ Flags: stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path. - --log.request.decision=LogFinishCall - Request Logging for logging the start and end - of requests. LogFinishCall is enabled by - default. LogFinishCall : Logs the finish call - of the requests. LogStartAndFinishCall : Logs - the start and finish call of the requests. - NoLogCall : Disable request logging. + --log.request.decision= Deprecation Warning - This flag would be soon + deprecated, and replaced with + `request.logging-config`. Request Logging for + logging the start and end of requests. By + default this flag is disabled. LogFinishCall: + Logs the finish call of the requests. + LogStartAndFinishCall: Logs the start and + finish call of the requests. NoLogCall: Disable + request logging. --objstore.config-file= Path to YAML file that contains object store configuration. See format details: @@ -400,6 +402,16 @@ Flags: happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: "", "SHA256". + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 ``` diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 0fc691d066..e0b0498f3a 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -140,6 +140,16 @@ Flags: --reloader.retry-interval=5s Controls how often reloader retries config reload in case of error. + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 --objstore.config-file= Path to YAML file that contains object store configuration. See format details: diff --git a/docs/components/store.md b/docs/components/store.md index db63a298f5..63a82b01f0 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -201,6 +201,16 @@ Flags: stripped prefix value in X-Forwarded-Prefix header. This allows thanos UI to be served on a sub-path. + --request.logging-config-file= + Path to YAML file with request logging + configuration. See format details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 + --request.logging-config= + Alternative to 'request.logging-config-file' + flag (lower priority). Content of YAML file + with request logging configuration. See format + details: + https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825 ``` diff --git a/go.mod b/go.mod index e3ce8199d3..7f12c9aba0 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,8 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 github.com/googleapis/gax-go v2.0.2+incompatible - github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/kit/v2 v2.0.0-20201002093600-73cf2ae9d891 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/golang-lru v0.5.4 github.com/jpillora/backoff v1.0.0 diff --git a/go.sum b/go.sum index f530ac84c6..a4f2090570 100644 --- a/go.sum +++ b/go.sum @@ -655,6 +655,11 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware/providers/kit/v2 v2.0.0-20201002093600-73cf2ae9d891 h1:RhOqTAECcPnehv3hKlYy1fAnpQ7rnZu58l3mpq6gT1k= +github.com/grpc-ecosystem/go-grpc-middleware/providers/kit/v2 v2.0.0-20201002093600-73cf2ae9d891/go.mod h1:516cTXxZzi4NBUBbKcwmO4Eqbb6GHAEd3o4N+GYyCBY= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea/go.mod h1:GugMBs30ZSAkckqXEAIEGyYdDH6EgqowG8ppA3Zt+AY= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 h1:guQyUpELu4I0wKgdsRBZDA5blfGiUleuppRSVy9Qbi0= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/go.mod h1:GhphxcdlaRyAuBSvo6rV71BvQcvB/vuX8ugCyybuS2k= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= diff --git a/pkg/extgrpc/client.go b/pkg/extgrpc/client.go index 5262b76fed..58305b32ef 100644 --- a/pkg/extgrpc/client.go +++ b/pkg/extgrpc/client.go @@ -8,7 +8,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/extkingpin/flags.go b/pkg/extkingpin/flags.go index 05dbd25563..481b2472ab 100644 --- a/pkg/extkingpin/flags.go +++ b/pkg/extkingpin/flags.go @@ -68,6 +68,17 @@ func RegisterCommonTracingFlags(app FlagClause) *extflag.PathOrContent { ) } +// RegisterRequestLoggingFlags registers flags to pass a request logging configuration to be used. +func RegisterRequestLoggingFlags(app FlagClause) *extflag.PathOrContent { + return extflag.RegisterPathOrContent( + app, + "request.logging-config", + // TODO @yashrsharma44: Change the link with the documented link for yaml configuration. + "YAML file with request logging configuration. See format details: https://gist.github.com/yashrsharma44/02f5765c5710dd09ce5d14e854f22825", + false, + ) +} + // RegisterSelectorRelabelFlags register flags to specify relabeling configuration selecting blocks to process. func RegisterSelectorRelabelFlags(cmd FlagClause) *extflag.PathOrContent { return extflag.RegisterPathOrContent( diff --git a/pkg/logging/grpc.go b/pkg/logging/grpc.go new file mode 100644 index 0000000000..c949c02ef4 --- /dev/null +++ b/pkg/logging/grpc.go @@ -0,0 +1,207 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package logging + +import ( + "fmt" + "math/rand" + "sort" + "strings" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" + "github.com/oklog/ulid" + "google.golang.org/grpc/status" + "gopkg.in/yaml.v2" +) + +// NewRequestConfig parses the string into a req logging config structure. +// Raise an error if unmarshalling is not possible, or values are not valid. +func NewRequestConfig(configYAML []byte) (*RequestConfig, error) { + reqLogConfig := &RequestConfig{} + if err := yaml.UnmarshalStrict(configYAML, reqLogConfig); err != nil { + return nil, err + } + return reqLogConfig, nil +} + +// checkOptionsConfigEmpty checks if the OptionsConfig struct is empty and valid. +// If invalid combination is present, return an error. +func checkOptionsConfigEmpty(optcfg OptionsConfig) (bool, error) { + if len(optcfg.Level) == 0 && !optcfg.Decision.LogEnd && !optcfg.Decision.LogStart { + return true, nil + } + if len(optcfg.Level) == 0 && (optcfg.Decision.LogStart || optcfg.Decision.LogEnd) { + return false, fmt.Errorf("level field is empty.") + } + return false, nil +} + +// fillGlobalOptionConfig configures all the method to have global config for logging. +func fillGlobalOptionConfig(reqLogConfig *RequestConfig, isgRPC bool) (string, bool, bool, error) { + globalLevel := "ERROR" + globalStart, globalEnd := false, false + + globalOptionConfig := reqLogConfig.Options + isEmpty, err := checkOptionsConfigEmpty(globalOptionConfig) + + // If the decision for logging is enabled with empty level field. + if err != nil { + return "", false, false, err + } + if !isEmpty { + globalLevel = globalOptionConfig.Level + globalStart = globalOptionConfig.Decision.LogStart + globalEnd = globalOptionConfig.Decision.LogEnd + } + + protocolOptionConfig := reqLogConfig.HTTP.Options + if isgRPC { + // gRPC config overrides the global config. + protocolOptionConfig = reqLogConfig.GRPC.Options + } + + isEmpty, err = checkOptionsConfigEmpty(protocolOptionConfig) + // If the decision for logging is enabled with empty level field. + if err != nil { + return "", false, false, err + } + + if !isEmpty { + globalLevel = protocolOptionConfig.Level + globalStart = protocolOptionConfig.Decision.LogStart + globalEnd = protocolOptionConfig.Decision.LogEnd + } + return globalLevel, globalStart, globalEnd, nil +} + +// getGRPCLoggingOption returns the logging ENUM based on logStart and logEnd values. +func getGRPCLoggingOption(logStart bool, logEnd bool) (grpc_logging.Decision, error) { + if !logStart && !logEnd { + return grpc_logging.NoLogCall, nil + } + if !logStart && logEnd { + return grpc_logging.LogFinishCall, nil + } + if logStart && logEnd { + return grpc_logging.LogStartAndFinishCall, nil + } + return -1, fmt.Errorf("log start call is not supported.") +} + +// validateLevel validates the list of level entries. +// Raise an error if empty or log level not in uppercase. +func validateLevel(level string) error { + if len(level) == 0 { + return fmt.Errorf("level field in YAML file is empty.") + } + if level == "INFO" || level == "DEBUG" || level == "ERROR" || level == "WARNING" { + return nil + } + return fmt.Errorf("The format of level is invalid. Expected INFO/DEBUG/ERROR/WARNING, got this %v", level) +} + +// NewGRPCOption adds in the config options and returns tags for logging middleware. +func NewGRPCOption(configYAML []byte) ([]tags.Option, []grpc_logging.Option, error) { + + // Configure tagOpts and logOpts. + tagOpts := []tags.Option{ + tags.WithFieldExtractor(func(_ string, req interface{}) map[string]string { + tagMap := tags.TagBasedRequestFieldExtractor("request-id")("", req) + // If a request-id exists for a given request. + if tagMap != nil { + if _, ok := tagMap["request-id"]; ok { + return tagMap + } + } + entropy := ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0) + reqID := ulid.MustNew(ulid.Timestamp(time.Now()), entropy) + tagMap = make(map[string]string) + tagMap["request-id"] = reqID.String() + return tagMap + }), + } + logOpts := []grpc_logging.Option{ + grpc_logging.WithDecider(func(_ string, _ error) grpc_logging.Decision { + return grpc_logging.NoLogCall + }), + grpc_logging.WithLevels(DefaultCodeToLevelGRPC), + } + + // Unmarshal YAML. + // if req logging is disabled. + if len(configYAML) == 0 { + return tagOpts, logOpts, nil + } + + reqLogConfig, err := NewRequestConfig(configYAML) + // If unmarshalling is an issue. + if err != nil { + return tagOpts, logOpts, err + } + + globalLevel, globalStart, globalEnd, err := fillGlobalOptionConfig(reqLogConfig, true) + // If global options have invalid entries. + if err != nil { + return tagOpts, logOpts, err + } + + // If the level entry does not matches our entries. + if err := validateLevel(globalLevel); err != nil { + return tagOpts, logOpts, err + } + + // If the combination is valid, use them, otherwise return error. + reqLogDecision, err := getGRPCLoggingOption(globalStart, globalEnd) + if err != nil { + return tagOpts, logOpts, err + } + + if len(reqLogConfig.GRPC.Config) == 0 { + logOpts = []grpc_logging.Option{ + grpc_logging.WithDecider(func(_ string, err error) grpc_logging.Decision { + + runtimeLevel := grpc_logging.DefaultServerCodeToLevel(status.Code(err)) + for _, lvl := range MapAllowedLevels[globalLevel] { + if string(runtimeLevel) == strings.ToLower(lvl) { + return reqLogDecision + } + } + return grpc_logging.NoLogCall + }), + grpc_logging.WithLevels(DefaultCodeToLevelGRPC), + } + return tagOpts, logOpts, nil + } + + logOpts = []grpc_logging.Option{ + grpc_logging.WithLevels(DefaultCodeToLevelGRPC), + } + + methodNameSlice := []string{} + + for _, eachConfig := range reqLogConfig.GRPC.Config { + eachConfigMethodName := interceptors.FullMethod(eachConfig.Service, eachConfig.Method) + methodNameSlice = append(methodNameSlice, eachConfigMethodName) + } + + logOpts = append(logOpts, []grpc_logging.Option{ + grpc_logging.WithDecider(func(runtimeMethodName string, err error) grpc_logging.Decision { + + idx := sort.SearchStrings(methodNameSlice, runtimeMethodName) + if idx < len(methodNameSlice) && methodNameSlice[idx] == runtimeMethodName { + runtimeLevel := grpc_logging.DefaultServerCodeToLevel(status.Code(err)) + for _, lvl := range MapAllowedLevels[globalLevel] { + if string(runtimeLevel) == strings.ToLower(lvl) { + return reqLogDecision + } + } + } + return grpc_logging.NoLogCall + }), + }...) + return tagOpts, logOpts, nil +} diff --git a/pkg/logging/http.go b/pkg/logging/http.go index db5ee12527..305ca5a340 100644 --- a/pkg/logging/http.go +++ b/pkg/logging/http.go @@ -5,6 +5,8 @@ package logging import ( "fmt" + "sort" + "strings" "net/http" "time" @@ -20,17 +22,16 @@ type HTTPServerMiddleware struct { } func (m *HTTPServerMiddleware) preCall(start time.Time) { - level.Debug(m.logger).Log("http.start_time", start.String(), "msg", "started call") + logger := m.opts.filterLog(m.logger) + level.Debug(logger).Log("http.start_time", start.String(), "msg", "started call") } func (m *HTTPServerMiddleware) postCall(name string, start time.Time, wrapped *httputil.ResponseWriterWithStatus, r *http.Request) { status := wrapped.Status() - logger := log.With(m.logger, "http.method", name, "http.request.id", r.Header.Get("X-Request-ID"), "http.code", fmt.Sprintf("%d", status), - "http.time_ms", fmt.Sprintf("%v", durationToMilliseconds(time.Since(start)))) + logger := log.With(m.logger, "http.method", fmt.Sprintf("%s %s", r.Method, r.URL), "http.request_id", r.Header.Get("X-Request-ID"), "http.status_code", fmt.Sprintf("%d", status), + "http.time_ms", fmt.Sprintf("%v", durationToMilliseconds(time.Since(start))), "http.remote_addr", r.RemoteAddr, "thanos.method_name", name) - if status >= 500 && status < 600 { - logger = log.With(logger, "http.error", fmt.Sprintf("%v", status)) - } + logger = m.opts.filterLog(logger) m.opts.levelFunc(logger, status).Log("msg", "finished call") } @@ -38,7 +39,9 @@ func (m *HTTPServerMiddleware) HTTPMiddleware(name string, next http.Handler) ht return func(w http.ResponseWriter, r *http.Request) { wrapped := httputil.WrapResponseWriterWithStatus(w) start := time.Now() - decision := m.opts.shouldLog() + port := strings.Split(r.Host, ":")[1] + + decision := m.opts.shouldLog(fmt.Sprintf("%s:%s", r.URL, port), nil) switch decision { case NoLogCall: @@ -56,6 +59,7 @@ func (m *HTTPServerMiddleware) HTTPMiddleware(name string, next http.Handler) ht } } +// NewHTTPServerMiddleware returns an http middleware. func NewHTTPServerMiddleware(logger log.Logger, opts ...Option) *HTTPServerMiddleware { o := evaluateOpt(opts) return &HTTPServerMiddleware{ @@ -63,3 +67,108 @@ func NewHTTPServerMiddleware(logger log.Logger, opts ...Option) *HTTPServerMiddl opts: o, } } + +// getHTTPLoggingOption returns the logging ENUM based on logStart and logEnd values. +func getHTTPLoggingOption(logStart bool, logEnd bool) (Decision, error) { + if !logStart && !logEnd { + return NoLogCall, nil + } + if !logStart && logEnd { + return LogFinishCall, nil + } + if logStart && logEnd { + return LogStartAndFinishCall, nil + } + return -1, fmt.Errorf("log start call is not supported.") +} + +// getLevel returns the level based logger. +func getLevel(lvl string) level.Option { + switch lvl { + case "INFO": + return level.AllowInfo() + case "DEBUG": + return level.AllowDebug() + case "WARN": + return level.AllowWarn() + case "ERROR": + return level.AllowError() + default: + return level.AllowAll() + } +} + +// NewHTTPOption returns a http config option. +func NewHTTPOption(configYAML []byte) ([]Option, error) { + // Define a black config option. + logOpts := []Option{ + WithDecider(func(_ string, err error) Decision { + return NoLogCall + }), + } + + // If req logging is disabled. + if len(configYAML) == 0 { + return logOpts, nil + } + + reqLogConfig, err := NewRequestConfig(configYAML) + // If unmarshalling is an issue. + if err != nil { + return logOpts, err + } + + globalLevel, globalStart, globalEnd, err := fillGlobalOptionConfig(reqLogConfig, false) + + // If global options have invalid entries. + if err != nil { + return logOpts, err + } + // If the level entry does not matches our entries. + if err := validateLevel(globalLevel); err != nil { + // fmt.Printf("HTTP") + return logOpts, err + } + + // If the combination is valid, use them, otherwise return error. + reqLogDecision, err := getHTTPLoggingOption(globalStart, globalEnd) + if err != nil { + return logOpts, err + } + + logOpts = []Option{ + WithFilter(func(logger log.Logger) log.Logger { + return level.NewFilter(logger, getLevel(globalLevel)) + }), + WithLevels(DefaultCodeToLevel), + } + + if len(reqLogConfig.HTTP.Config) == 0 { + logOpts = append(logOpts, []Option{WithDecider(func(_ string, err error) Decision { + return reqLogDecision + }), + }...) + return logOpts, nil + } + + methodNameSlice := []string{} + + for _, eachConfig := range reqLogConfig.HTTP.Config { + eachConfigName := fmt.Sprintf("%v:%v", eachConfig.Path, eachConfig.Port) + methodNameSlice = append(methodNameSlice, eachConfigName) + } + + sort.Strings(methodNameSlice) + + logOpts = append(logOpts, []Option{ + WithDecider(func(runtimeMethodName string, err error) Decision { + idx := sort.SearchStrings(methodNameSlice, runtimeMethodName) + if idx < len(methodNameSlice) && methodNameSlice[idx] == runtimeMethodName { + return reqLogDecision + } + return NoLogCall + }), + }...) + return logOpts, nil + +} diff --git a/pkg/logging/options.go b/pkg/logging/options.go index a119c08ae2..9303075ae2 100644 --- a/pkg/logging/options.go +++ b/pkg/logging/options.go @@ -5,10 +5,16 @@ package logging import ( "fmt" + "math/rand" "time" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" + "github.com/oklog/ulid" + "github.com/thanos-io/thanos/pkg/extflag" + "google.golang.org/grpc/codes" ) // Decision defines rules for enabling start and end of logging. @@ -28,6 +34,7 @@ var defaultOptions = &options{ codeFunc: DefaultErrorToCode, levelFunc: DefaultCodeToLevel, durationFieldFunc: DurationToTimeMillisFields, + filterLog: DefaultFilterLogging, } func evaluateOpt(opts []Option) *options { @@ -54,6 +61,14 @@ func WithLevels(f CodeToLevel) Option { } } +// WithFilter customizes the function for deciding which level of logging should be allowed. +// Follows go-kit Allow convention. +func WithFilter(f FilterLogging) Option { + return func(o *options) { + o.filterLog = f + } +} + // Interface for the additional methods. // Types for the Options. @@ -66,17 +81,17 @@ type Fields []string // for the http response. type ErrorToCode func(err error) int +// DefaultErrorToCode returns an InternalServerError. func DefaultErrorToCode(_ error) int { return 500 } // Decider function defines rules for suppressing the logging. -// TODO : Add the method name as a parameter in case we want to log based on method names. -type Decider func() Decision +type Decider func(methodName string, err error) Decision // DefaultDeciderMethod is the default implementation of decider to see if you should log the call // by default this is set to LogStartAndFinishCall. -func DefaultDeciderMethod() Decision { +func DefaultDeciderMethod(_ string, _ error) Decision { return LogStartAndFinishCall } @@ -86,11 +101,20 @@ type CodeToLevel func(logger log.Logger, code int) log.Logger // DurationToFields function defines how to produce duration fields for logging. type DurationToFields func(duration time.Duration) Fields +// FilterLogging makes sure only the logs with level=lvl gets logged, or filtered. +type FilterLogging func(logger log.Logger) log.Logger + +// DefaultFilterLogging allows logs from all levels to be logged in output. +func DefaultFilterLogging(logger log.Logger) log.Logger { + return level.NewFilter(logger, level.AllowAll()) +} + type options struct { levelFunc CodeToLevel shouldLog Decider codeFunc ErrorToCode durationFieldFunc DurationToFields + filterLog FilterLogging } // DefaultCodeToLevel is the helper mapper that maps HTTP Response codes to log levels. @@ -101,6 +125,16 @@ func DefaultCodeToLevel(logger log.Logger, code int) log.Logger { return level.Error(logger) } +// DefaultCodeToLevelGRPC is the helper mapper that maps gRPC Response codes to log levels. +func DefaultCodeToLevelGRPC(c codes.Code) grpc_logging.Level { + switch c { + case codes.Unknown, codes.Unimplemented, codes.Internal, codes.DataLoss: + return grpc_logging.ERROR + default: + return grpc_logging.DEBUG + } +} + // DurationToTimeMillisFields converts the duration to milliseconds and uses the key `http.time_ms`. func DurationToTimeMillisFields(duration time.Duration) Fields { return Fields{"http.time_ms", fmt.Sprintf("%v", durationToMilliseconds(duration))} @@ -116,3 +150,101 @@ var LogDecision = map[string]Decision{ "LogFinishCall": LogFinishCall, "LogStartAndFinishCall": LogStartAndFinishCall, } + +// MapAllowedLevels allows to map a given level to a list of allowed level. +// Convention taken from go-kit/level v0.10.0 https://godoc.org/github.com/go-kit/kit/log/level#AllowAll. +var MapAllowedLevels = map[string][]string{ + "DEBUG": {"INFO", "DEBUG", "WARN", "ERROR"}, + "ERROR": {"ERROR"}, + "INFO": {"INFO", "WARN", "ERROR"}, + "WARN": {"WARN", "ERROR"}, +} + +// TODO: @yashrsharma44 - To be deprecated in the next release. +func ParseHTTPOptions(flagDecision string, reqLogConfig *extflag.PathOrContent) ([]Option, error) { + // Default Option: No Logging. + logOpts := []Option{WithDecider(func(_ string, _ error) Decision { + return NoLogCall + })} + + // If flag is incorrectly parsed. + configYAML, err := reqLogConfig.Content() + if err != nil { + return logOpts, fmt.Errorf("getting request logging config failed. %v", err) + } + + // Check if the user enables request logging through flags and YAML. + if len(configYAML) != 0 && len(flagDecision) != 0 { + return logOpts, fmt.Errorf("Both log.request.decision and request.logging has been enabled. Please use one of the flags!") + } + // If old flag is enabled. + if len(flagDecision) > 0 { + logOpts := []Option{WithDecider(func(_ string, _ error) Decision { + return LogDecision[flagDecision] + })} + return logOpts, nil + } + logOpts, err = NewHTTPOption(configYAML) + if err != nil { + return logOpts, err + } + + return logOpts, nil +} + +// TODO: @yashrsharma44 - To be deprecated in the next release. +func ParsegRPCOptions(flagDecision string, reqLogConfig *extflag.PathOrContent) ([]tags.Option, []grpc_logging.Option, error) { + // Default Option: No Logging. + logOpts := []grpc_logging.Option{grpc_logging.WithDecider(func(_ string, _ error) grpc_logging.Decision { + return grpc_logging.NoLogCall + })} + + configYAML, err := reqLogConfig.Content() + if err != nil { + return []tags.Option{}, logOpts, fmt.Errorf("getting request logging config failed. %v", err) + } + + // Check if the user enables request logging through flags and YAML. + if len(configYAML) != 0 && len(flagDecision) != 0 { + return []tags.Option{}, logOpts, fmt.Errorf("Both log.request.decision and request.logging-config has been enabled. Please use one of the flags!") + } + + // If the old flag is empty, use the new YAML config. + if len(flagDecision) == 0 { + tagOpts, logOpts, err := NewGRPCOption(configYAML) + if err != nil { + return []tags.Option{}, logOpts, err + } + return tagOpts, logOpts, nil + } + + tagOpts := []tags.Option{ + tags.WithFieldExtractor(func(_ string, req interface{}) map[string]string { + tagMap := tags.TagBasedRequestFieldExtractor("request-id")("", req) + // If a request-id exists for a given request. + if tagMap != nil { + if _, ok := tagMap["request-id"]; ok { + return tagMap + } + } + entropy := ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0) + reqID := ulid.MustNew(ulid.Timestamp(time.Now()), entropy) + tagMap = make(map[string]string) + tagMap["request-id"] = reqID.String() + return tagMap + }), + } + logOpts = []grpc_logging.Option{grpc_logging.WithDecider(func(_ string, _ error) grpc_logging.Decision { + switch flagDecision { + case "NoLogCall": + return grpc_logging.NoLogCall + case "LogFinishCall": + return grpc_logging.LogFinishCall + case "LogStartAndFinishCall": + return grpc_logging.LogStartAndFinishCall + default: + return grpc_logging.NoLogCall + } + })} + return tagOpts, logOpts, nil +} diff --git a/pkg/logging/yaml_parser.go b/pkg/logging/yaml_parser.go new file mode 100644 index 0000000000..ed3c893ed9 --- /dev/null +++ b/pkg/logging/yaml_parser.go @@ -0,0 +1,40 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package logging + +type RequestConfig struct { + HTTP HTTPProtocolConfigs `yaml:"http"` + GRPC GRPCProtocolConfigs `yaml:"grpc"` + Options OptionsConfig `yaml:"options"` +} + +type HTTPProtocolConfigs struct { + Options OptionsConfig `yaml:"options"` + Config []HTTPProtocolConfig `yaml:"config"` +} + +type GRPCProtocolConfigs struct { + Options OptionsConfig `yaml:"options"` + Config []GRPCProtocolConfig `yaml:"config"` +} + +type OptionsConfig struct { + Level string `yaml:"level"` + Decision DecisionConfig `yaml:"decision"` +} + +type DecisionConfig struct { + LogStart bool `yaml:"log_start"` + LogEnd bool `yaml:"log_end"` +} + +type HTTPProtocolConfig struct { + Path string `yaml:"path"` + Port uint64 `yaml:"port"` +} + +type GRPCProtocolConfig struct { + Service string `yaml:"service"` + Method string `yaml:"method"` +} diff --git a/pkg/server/grpc/grpc.go b/pkg/server/grpc/grpc.go index 5976451c8f..0d029b20f7 100644 --- a/pkg/server/grpc/grpc.go +++ b/pkg/server/grpc/grpc.go @@ -11,8 +11,11 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + kit "github.com/grpc-ecosystem/go-grpc-middleware/providers/kit/v2" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tags" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -43,7 +46,7 @@ type Server struct { // New creates a new gRPC Store API. // If rulesSrv is not nil, it also registers Rules API to the returned server. -func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, comp component.Component, probe *prober.GRPCProbe, opts ...Option) *Server { +func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer, logOpts []grpc_logging.Option, tagsOpts []tags.Option, comp component.Component, probe *prober.GRPCProbe, opts ...Option) *Server { logger = log.With(logger, "service", "gRPC/server", "component", comp.String()) options := options{ network: "tcp", @@ -71,13 +74,17 @@ func New(logger log.Logger, reg prometheus.Registerer, tracer opentracing.Tracer grpc.MaxSendMsgSize(math.MaxInt32), grpc_middleware.WithUnaryServerChain( met.UnaryServerInterceptor(), + tags.UnaryServerInterceptor(tagsOpts...), tracing.UnaryServerInterceptor(tracer), grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + grpc_logging.UnaryServerInterceptor(kit.InterceptorLogger(logger), logOpts...), ), grpc_middleware.WithStreamServerChain( met.StreamServerInterceptor(), + tags.StreamServerInterceptor(tagsOpts...), tracing.StreamServerInterceptor(tracer), grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + grpc_logging.StreamServerInterceptor(kit.InterceptorLogger(logger), logOpts...), ), }...) diff --git a/pkg/store/multitsdb.go b/pkg/store/multitsdb.go index 902611e658..d57753eb6a 100644 --- a/pkg/store/multitsdb.go +++ b/pkg/store/multitsdb.go @@ -10,7 +10,7 @@ import ( "sync" "github.com/go-kit/kit/log" - grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tracing" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 4391a36619..20044a2e62 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -14,7 +14,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tracing" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/tracing/grpc.go b/pkg/tracing/grpc.go index 303c387a67..78b4391bf8 100644 --- a/pkg/tracing/grpc.go +++ b/pkg/tracing/grpc.go @@ -6,8 +6,8 @@ package tracing import ( "context" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" + grpc_opentracing "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/tracing" opentracing "github.com/opentracing/opentracing-go" "google.golang.org/grpc" ) diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 5570234c8f..473b0ca8f6 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -21,6 +21,7 @@ import ( "github.com/thanos-io/thanos/pkg/alert" "github.com/thanos-io/thanos/pkg/cacheutil" http_util "github.com/thanos-io/thanos/pkg/http" + "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" @@ -49,6 +50,7 @@ var ( client.ALIYUNOSS: oss.Config{}, client.FILESYSTEM: filesystem.Config{}, } + tracingConfigs = map[trclient.TracingProvider]interface{}{ trclient.JAEGER: jaeger.Config{}, trclient.STACKDRIVER: stackdriver.Config{}, @@ -84,6 +86,12 @@ func main() { } } + if err := generate(logging.RequestConfig{}, generateName("logging_", "request"), *outputDir); err != nil { + level.Error(logger).Log("msg", "failed to generate", "type", "request_logging", "err", err) + os.Exit(1) + + } + for typ, config := range tracingConfigs { if err := generate(trclient.TracingConfig{Type: typ, Config: config}, generateName("tracing_", string(typ)), *outputDir); err != nil { level.Error(logger).Log("msg", "failed to generate", "type", typ, "err", err)