From d1e06967987043ea2ef96595454b2014bdff66aa Mon Sep 17 00:00:00 2001 From: Mark Farrell Date: Mon, 22 Apr 2024 10:01:44 +1000 Subject: [PATCH 1/2] add vault.agent.authenticated metric fix metric name --- changelog/26570.txt | 3 ++ command/agentproxyshared/auth/auth.go | 37 +++++++++++++++++++ .../docs/agent-and-proxy/agent/index.mdx | 2 + 3 files changed, 42 insertions(+) create mode 100644 changelog/26570.txt diff --git a/changelog/26570.txt b/changelog/26570.txt new file mode 100644 index 000000000000..79efe31414f9 --- /dev/null +++ b/changelog/26570.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: Add metric (vault.agent.authenticated) that is set to 1 when vault agent has a valid token and zero if it does not. +``` \ No newline at end of file diff --git a/command/agentproxyshared/auth/auth.go b/command/agentproxyshared/auth/auth.go index 49ae395c48da..f08737342ca1 100644 --- a/command/agentproxyshared/auth/auth.go +++ b/command/agentproxyshared/auth/auth.go @@ -144,12 +144,18 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { backoffCfg := newAutoAuthBackoff(ah.minBackoff, ah.maxBackoff, ah.exitOnError) ah.logger.Info("starting auth handler") + + // Set unauthenticated when starting up + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) + defer func() { am.Shutdown() close(ah.OutputCh) close(ah.TemplateTokenCh) close(ah.ExecTokenCh) ah.logger.Info("auth handler stopped") + // Set unauthenticated when shutting down + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) }() credCh := am.NewCreds() @@ -216,6 +222,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("error creating client for authentication call", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -243,6 +251,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("could not look up token", "err", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -263,6 +273,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -276,6 +288,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -314,6 +327,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("error authenticating", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -329,6 +344,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if secret.WrapInfo == nil { ah.logger.Error("authentication returned nil wrap info", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -338,6 +355,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if secret.WrapInfo.Token == "" { ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -348,6 +367,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -387,6 +408,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if secret == nil || secret.Data == nil { ah.logger.Error("token file validation failed, token may be invalid", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -397,6 +420,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if !ok || token == "" { ah.logger.Error("token file validation returned empty client token", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -413,6 +438,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { Renewable: renewable, } ah.logger.Info("authentication successful, sending token to sinks") + ah.OutputCh <- token if ah.enableTemplateTokenCh { ah.TemplateTokenCh <- token @@ -429,6 +455,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if secret == nil || secret.Auth == nil { ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -438,6 +466,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if secret.Auth.ClientToken == "" { ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -470,6 +500,8 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { if err != nil { ah.logger.Error("error creating lifetime watcher", "error", err, "backoff", backoffCfg) metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) if backoffSleep(ctx, backoffCfg) { continue @@ -478,6 +510,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "success"}, 1) + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 1) // We don't want to trigger the renewal process for tokens with // unlimited TTL, such as the root token. if leaseDuration == 0 && isTokenFileMethod { @@ -499,12 +532,16 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { ah.logger.Info("lifetime watcher done channel triggered") if err != nil { metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) + // Set unauthenticated when authentication fails + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) ah.logger.Error("error renewing token", "error", err) } break LifetimeWatcherLoop case <-watcher.RenewCh(): metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "success"}, 1) + // Set authenticated when authentication succeeds + metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 1) ah.logger.Info("renewed auth token") case <-credCh: diff --git a/website/content/docs/agent-and-proxy/agent/index.mdx b/website/content/docs/agent-and-proxy/agent/index.mdx index 21a8ba286340..eb77ea534059 100644 --- a/website/content/docs/agent-and-proxy/agent/index.mdx +++ b/website/content/docs/agent-and-proxy/agent/index.mdx @@ -288,6 +288,8 @@ runtime metrics about its performance, the auto-auth and the cache status: | Metric | Description | Type | | -------------------------------- | ---------------------------------------------------- | ------- | +| `vault.agent.authenticated` | Current authentication status (1 - has valid token, | gauge | +| | 0 - no valid token) | | | `vault.agent.auth.failure` | Number of authentication failures | counter | | `vault.agent.auth.success` | Number of authentication successes | counter | | `vault.agent.proxy.success` | Number of requests successfully proxied | counter | From 4f6f8e7038ae3a34171e36add19ad9d3fd4396ed Mon Sep 17 00:00:00 2001 From: Violet Hynes Date: Tue, 28 May 2024 11:51:55 -0400 Subject: [PATCH 2/2] Update command/agentproxyshared/auth/auth.go --- command/agentproxyshared/auth/auth.go | 1 - 1 file changed, 1 deletion(-) diff --git a/command/agentproxyshared/auth/auth.go b/command/agentproxyshared/auth/auth.go index 6bd9d02c5994..bcfbf3be9c5e 100644 --- a/command/agentproxyshared/auth/auth.go +++ b/command/agentproxyshared/auth/auth.go @@ -535,7 +535,6 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { metrics.IncrCounter([]string{ah.metricsSignifier, "auth", "failure"}, 1) // Set unauthenticated when authentication fails metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 0) - ah.logger.Error("error renewing token", "error", err) // Add some exponential backoff so that if auth is successful // but the watcher errors, we won't go into an immediate