diff --git a/plugins/common/http/config.go b/plugins/common/http/config.go index ffd24558be559..3f5bfdfa5f4a2 100644 --- a/plugins/common/http/config.go +++ b/plugins/common/http/config.go @@ -19,10 +19,11 @@ import ( // Common HTTP client struct. type HTTPClientConfig struct { - Timeout config.Duration `toml:"timeout"` - IdleConnTimeout config.Duration `toml:"idle_conn_timeout"` - MaxIdleConns int `toml:"max_idle_conn"` - MaxIdleConnsPerHost int `toml:"max_idle_conn_per_host"` + Timeout config.Duration `toml:"timeout"` + IdleConnTimeout config.Duration `toml:"idle_conn_timeout"` + MaxIdleConns int `toml:"max_idle_conn"` + MaxIdleConnsPerHost int `toml:"max_idle_conn_per_host"` + ResponseHeaderTimeout config.Duration `toml:"response_timeout"` proxy.HTTPProxy tls.ClientConfig @@ -42,11 +43,12 @@ func (h *HTTPClientConfig) CreateClient(ctx context.Context, log telegraf.Logger } transport := &http.Transport{ - TLSClientConfig: tlsCfg, - Proxy: prox, - IdleConnTimeout: time.Duration(h.IdleConnTimeout), - MaxIdleConns: h.MaxIdleConns, - MaxIdleConnsPerHost: h.MaxIdleConnsPerHost, + TLSClientConfig: tlsCfg, + Proxy: prox, + IdleConnTimeout: time.Duration(h.IdleConnTimeout), + MaxIdleConns: h.MaxIdleConns, + MaxIdleConnsPerHost: h.MaxIdleConnsPerHost, + ResponseHeaderTimeout: time.Duration(h.ResponseHeaderTimeout), } // Register "http+unix" and "https+unix" protocol handler. diff --git a/plugins/inputs/vault/testdata/response_integration_1.13.3.influx b/plugins/inputs/vault/testdata/response_integration_1.13.3.influx new file mode 100644 index 0000000000000..1f628d0aaed6a --- /dev/null +++ b/plugins/inputs/vault/testdata/response_integration_1.13.3.influx @@ -0,0 +1,51 @@ +vault.audit.log_request_failure count=3i,max=0i,mean=0,min=0i,rate=0,stddev=0,sum=0i 1697798140000000000 +vault.audit.log_response_failure count=3i,max=0i,mean=0,min=0i,rate=0,stddev=0,sum=0i 1697798140000000000 +vault.barrier.estimated_encryptions,term=1 count=33i,max=1i,mean=1,min=1i,rate=3.3,stddev=0,sum=33i 1697798140000000000 +vault.cache.hit count=15i,max=1i,mean=1,min=1i,rate=1.5,stddev=0,sum=15i 1697798140000000000 +vault.cache.miss count=54i,max=1i,mean=1,min=1i,rate=5.4,stddev=0,sum=54i 1697798140000000000 +vault.cache.write count=28i,max=1i,mean=1,min=1i,rate=2.8,stddev=0,sum=28i 1697798140000000000 +vault.seal.decrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000 +vault.seal.encrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000 +vault.seal.shamir.decrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000 +vault.seal.shamir.encrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000 +vault.token.create_root count=2i,max=1i,mean=1,min=1i,rate=0.2,stddev=0,sum=2i 1697798140000000000 +vault.token.creation,auth_method=token,cluster=vault-cluster-57e15bbb,creation_ttl=1m,mount_point=auth/token/,namespace=root,token_type=service count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000 +vault.core.locked_users value=0i 1697798140000000000 +vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=false,type=auth value=1i 1697798140000000000 +vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=true,type=auth value=0i 1697798140000000000 +vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=false,type=logical value=3i 1697798140000000000 +vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=true,type=logical value=1i 1697798140000000000 +vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=false,type=auth value=251i 1697798140000000000 +vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=true,type=auth value=56i 1697798140000000000 +vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=false,type=logical value=556i 1697798140000000000 +vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=true,type=logical value=302i 1697798140000000000 +vault.audit.log_request count=3i,max=0.00892999954521656,mean=0.005283333205928405,min=0.0029299999587237835,rate=0.0015849999617785215,stddev=0.0032022697614016125,sum=0.015849999617785215 1697798140000000000 +vault.audit.log_response count=3i,max=0.006140000186860561,mean=0.004420000050837795,min=0.00343999988399446,rate=0.0013260000152513385,stddev=0.0014943896328128183,sum=0.013260000152513385 1697798140000000000 +vault.barrier.delete count=4i,max=0.004600000102072954,mean=0.003989999939221889,min=0.0035099999513477087,rate=0.0015959999756887556,stddev=0.00045658157079035244,sum=0.015959999756887555 1697798140000000000 +vault.barrier.get count=75i,max=0.049591001123189926,mean=0.00808781327912584,min=0.0007999999797903001,rate=0.060658599593443795,stddev=0.010314610354373298,sum=0.606585995934438 1697798140000000000 +vault.barrier.list count=37i,max=0.0049310000613331795,mean=0.0013254594665045875,min=0.0002800000074785203,rate=0.004904200026066974,stddev=0.0010832305178496164,sum=0.04904200026066974 1697798140000000000 +vault.barrier.put count=31i,max=0.05448000133037567,mean=0.013786516182364957,min=0.0061900001019239426,rate=0.042738200165331364,stddev=0.008650664562076636,sum=0.42738200165331364 1697798140000000000 +vault.core.check_token count=3i,max=1.133538007736206,mean=0.4793433391799529,min=0.02119000069797039,rate=0.1438030017539859,stddev=0.5815098232303996,sum=1.4380300175398588 1697798140000000000 +vault.core.fetch_acl_and_token count=3i,max=0.010940000414848328,mean=0.00783000017205874,min=0.006060000043362379,rate=0.0023490000516176225,stddev=0.0027019069443178717,sum=0.023490000516176224 1697798140000000000 +vault.core.handle_request count=3i,max=1.790362000465393,mean=1.1120976607004802,min=0.525954008102417,rate=0.33362929821014403,stddev=0.6372178266032051,sum=3.3362929821014404 1697798140000000000 +vault.core.post_unseal count=2i,max=171.49169921875,mean=86.97485756874084,min=2.4580159187316895,rate=17.394971513748168,stddev=119.52486371038222,sum=173.9497151374817 1697798140000000000 +vault.core.pre_seal count=1i,max=0.46343299746513367,mean=0.46343299746513367,min=0.46343299746513367,rate=0.04634329974651337,stddev=0,sum=0.46343299746513367 1697798140000000000 +vault.expire.register-auth count=1i,max=0.2956019937992096,mean=0.2956019937992096,min=0.2956019937992096,rate=0.029560199379920958,stddev=0,sum=0.2956019937992096 1697798140000000000 +vault.expire.revoke count=1i,max=0.39290300011634827,mean=0.39290300011634827,min=0.39290300011634827,rate=0.039290300011634825,stddev=0,sum=0.39290300011634827 1697798140000000000 +vault.expire.revoke-by-token count=1i,max=0.14829100668430328,mean=0.14829100668430328,min=0.14829100668430328,rate=0.014829100668430328,stddev=0,sum=0.14829100668430328 1697798140000000000 +vault.expire.revoke-common count=2i,max=0.3899630010128021,mean=0.25605200231075287,min=0.12214100360870361,rate=0.05121040046215057,stddev=0.18937875051536399,sum=0.5121040046215057 1697798140000000000 +vault.policy.get_policy count=11i,max=0.46198299527168274,mean=0.056309545021080834,min=0.001560000004246831,rate=0.06194049952318892,stddev=0.13695683947079185,sum=0.6194049952318892 1697798140000000000 +vault.route.update.auth-token- count=2i,max=0.4904730021953583,mean=0.3715669959783554,min=0.25266098976135254,rate=0.07431339919567108,stddev=0.168158486639705,sum=0.7431339919567108 1697798140000000000 +vault.route.update.sys- count=1i,max=0.6211339831352234,mean=0.6211339831352234,min=0.6211339831352234,rate=0.06211339831352234,stddev=0,sum=0.6211339831352234 1697798140000000000 +vault.seal.decrypt.time count=1i,max=0.019300000742077827,mean=0.019300000742077827,min=0.019300000742077827,rate=0.0019300000742077828,stddev=0,sum=0.019300000742077827 1697798140000000000 +vault.seal.encrypt.time count=1i,max=0.008750000037252903,mean=0.008750000037252903,min=0.008750000037252903,rate=0.0008750000037252903,stddev=0,sum=0.008750000037252903 1697798140000000000 +vault.seal.shamir.decrypt.time count=1i,max=0.04343099892139435,mean=0.04343099892139435,min=0.04343099892139435,rate=0.004343099892139435,stddev=0,sum=0.04343099892139435 1697798140000000000 +vault.seal.shamir.encrypt.time count=1i,max=0.25516200065612793,mean=0.25516200065612793,min=0.25516200065612793,rate=0.025516200065612792,stddev=0,sum=0.25516200065612793 1697798140000000000 +vault.token.create count=2i,max=0.7710049748420715,mean=0.42180248722434044,min=0.07259999960660934,rate=0.0843604974448681,stddev=0.4938468940034181,sum=0.8436049744486809 1697798140000000000 +vault.token.createAccessor count=2i,max=0.032510001212358475,mean=0.02841000072658062,min=0.024310000240802765,rate=0.005682000145316124,stddev=0.00579827629272332,sum=0.05682000145316124 1697798140000000000 +vault.token.lookup count=6i,max=0.08506099879741669,mean=0.04382349985341231,min=0.027969999238848686,rate=0.026294099912047387,stddev=0.021177076886439525,sum=0.26294099912047386 1697798140000000000 +vault.token.revoke-tree count=1i,max=0.2888520061969757,mean=0.2888520061969757,min=0.2888520061969757,rate=0.02888520061969757,stddev=0,sum=0.2888520061969757 1697798140000000000 +vault.token.store count=1i,max=0.023420000448822975,mean=0.023420000448822975,min=0.023420000448822975,rate=0.0023420000448822974,stddev=0,sum=0.023420000448822975 1697798140000000000 +vault.core.unseal count=1i,max=2.752518892288208,mean=2.752518892288208,min=2.752518892288208,rate=0.2752518892288208,stddev=0,sum=2.752518892288208 1697798140000000000 +vault.core.unsealed,cluster=a value=0i 1697798140000000000 +vault.core.unsealed,cluster=vault-cluster-57e15bbb value=1i 1697798140000000000 diff --git a/plugins/inputs/vault/vault.go b/plugins/inputs/vault/vault.go index c73dece4092d1..f9a14ed074832 100644 --- a/plugins/inputs/vault/vault.go +++ b/plugins/inputs/vault/vault.go @@ -2,6 +2,7 @@ package vault import ( + "context" _ "embed" "encoding/json" "fmt" @@ -13,7 +14,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" - "github.com/influxdata/telegraf/plugins/common/tls" + httpcommon "github.com/influxdata/telegraf/plugins/common/http" "github.com/influxdata/telegraf/plugins/inputs" ) @@ -22,28 +23,17 @@ var sampleConfig string // Vault configuration object type Vault struct { - URL string `toml:"url"` + URL string `toml:"url"` + TokenFile string `toml:"token_file"` + Token string `toml:"token"` + Log telegraf.Logger `toml:"-"` + httpcommon.HTTPClientConfig - TokenFile string `toml:"token_file"` - Token string `toml:"token"` - - ResponseTimeout config.Duration `toml:"response_timeout"` - - tls.ClientConfig - - roundTripper http.RoundTripper + client *http.Client } const timeLayout = "2006-01-02 15:04:05 -0700 MST" -func init() { - inputs.Add("vault", func() telegraf.Input { - return &Vault{ - ResponseTimeout: config.Duration(5 * time.Second), - } - }) -} - func (*Vault) SampleConfig() string { return sampleConfig } @@ -69,16 +59,12 @@ func (n *Vault) Init() error { n.Token = strings.TrimSpace(string(token)) } - tlsCfg, err := n.ClientConfig.TLSConfig() + ctx := context.Background() + client, err := n.HTTPClientConfig.CreateClient(ctx, n.Log) if err != nil { - return fmt.Errorf("setting up TLS configuration failed: %w", err) - } - - n.roundTripper = &http.Transport{ - TLSHandshakeTimeout: 5 * time.Second, - TLSClientConfig: tlsCfg, - ResponseHeaderTimeout: time.Duration(n.ResponseTimeout), + return fmt.Errorf("creating client failed: %w", err) } + n.client = client return nil } @@ -102,7 +88,7 @@ func (n *Vault) loadJSON(url string) (*SysMetrics, error) { req.Header.Set("X-Vault-Token", n.Token) req.Header.Add("Accept", "application/json") - resp, err := n.roundTripper.RoundTrip(req) + resp, err := n.client.Do(req) if err != nil { return nil, fmt.Errorf("error making HTTP request to %q: %w", url, err) } @@ -191,3 +177,13 @@ func buildVaultMetrics(acc telegraf.Accumulator, sysMetrics *SysMetrics) error { return nil } + +func init() { + inputs.Add("vault", func() telegraf.Input { + return &Vault{ + HTTPClientConfig: httpcommon.HTTPClientConfig{ + ResponseHeaderTimeout: config.Duration(5 * time.Second), + }, + } + }) +} diff --git a/plugins/inputs/vault/vault_test.go b/plugins/inputs/vault/vault_test.go index 67c7a01550b9c..b0d051c79d2c7 100644 --- a/plugins/inputs/vault/vault_test.go +++ b/plugins/inputs/vault/vault_test.go @@ -5,12 +5,20 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "testing" "time" + dockercontainer "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/wait" ) func TestVaultStats(t *testing.T) { @@ -96,3 +104,142 @@ func TestVaultStats(t *testing.T) { }) } } + +func TestRedirect(t *testing.T) { + expected := []telegraf.Metric{ + testutil.MustMetric( + "vault.raft.replication.appendEntries.logs", + map[string]string{ + "peer_id": "clustnode-02", + }, + map[string]interface{}{ + "count": int(130), + "rate": float64(0.2), + "sum": int(2), + "min": int(0), + "max": int(1), + "mean": float64(0.015384615384615385), + "stddev": float64(0.12355304447984486), + }, + time.Unix(1638287340, 0), + 1, + ), + testutil.MustMetric( + "vault.core.unsealed", + map[string]string{ + "cluster": "vault-cluster-23b671c7", + }, + map[string]interface{}{ + "value": int(1), + }, + time.Unix(1638287340, 0), + 2, + ), + testutil.MustMetric( + "vault.token.lookup", + map[string]string{}, + map[string]interface{}{ + "count": int(5135), + "max": float64(16.22449493408203), + "mean": float64(0.1698389152269865), + "min": float64(0.06690400093793869), + "rate": float64(87.21228296905755), + "stddev": float64(0.24637634000854705), + "sum": float64(872.1228296905756), + }, + time.Unix(1638287340, 0), + 1, + ), + } + + response, err := os.ReadFile("testdata/response_key_metrics.json") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/v1/sys/metrics": + redirectURL := "http://" + r.Host + "/custom/metrics" + http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) + case "/custom/metrics": + w.WriteHeader(http.StatusOK) + _, _ = w.Write(response) + } + })) + defer server.Close() + + // Setup the plugin + plugin := &Vault{ + URL: server.URL, + Token: "s.CDDrgg5zPv5ssI0Z2P4qxJj2", + } + require.NoError(t, plugin.Init()) + + var acc testutil.Accumulator + require.NoError(t, plugin.Gather(&acc)) + actual := acc.GetTelegrafMetrics() + testutil.RequireMetricsEqual(t, expected, actual) +} + +func TestIntegration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Start the docker container + container := testutil.Container{ + Image: "vault:1.13.3", + ExposedPorts: []string{"8200"}, + Env: map[string]string{ + "VAULT_DEV_ROOT_TOKEN_ID": "root", + }, + HostConfigModifier: func(hc *dockercontainer.HostConfig) { + hc.CapAdd = []string{"IPC_LOCK"} + }, + WaitingFor: wait.ForAll( + wait.ForLog("Root Token: root"), + wait.ForListeningPort(nat.Port("8200")), + ), + } + require.NoError(t, container.Start(), "failed to start container") + defer container.Terminate() + + // Setup the plugin + port := container.Ports["8200"] + plugin := &Vault{ + URL: "http://" + container.Address + ":" + port, + Token: "root", + } + require.NoError(t, plugin.Init()) + + // Setup the expectations + buf, err := os.ReadFile(filepath.Join("testdata", "response_integration_1.13.3.influx")) + require.NoError(t, err) + parser := &influx.Parser{} + require.NoError(t, parser.Init()) + raw, err := parser.Parse(buf) + require.NoError(t, err) + expected := make([]telegraf.Metric, 0, len(raw)) + for _, r := range raw { + vt := telegraf.Counter + switch r.Name() { + case "vault.core.locked_users", "vault.core.mount_table.num_entries", + "vault.core.mount_table.size", "vault.core.unsealed": + vt = telegraf.Gauge + } + m := metric.New(r.Name(), r.Tags(), r.Fields(), r.Time(), vt) + expected = append(expected, m) + } + + options := []cmp.Option{ + testutil.SortMetrics(), + testutil.IgnoreTags("cluster"), + testutil.IgnoreTime(), + } + + // Collect the metrics and compare + var acc testutil.Accumulator + require.NoError(t, plugin.Gather(&acc)) + + actual := acc.GetTelegrafMetrics() + testutil.RequireMetricsStructureEqual(t, expected, actual, options...) +} diff --git a/testutil/container.go b/testutil/container.go index a63ec6b602e48..f80ea5a04a8bf 100644 --- a/testutil/container.go +++ b/testutil/container.go @@ -8,6 +8,7 @@ import ( "io" "strings" + dockercontainer "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -22,15 +23,16 @@ func (g *TestLogConsumer) Accept(l testcontainers.Log) { } type Container struct { - BindMounts map[string]string - Entrypoint []string - Env map[string]string - ExposedPorts []string - Cmd []string - Image string - Name string - Networks []string - WaitingFor wait.Strategy + BindMounts map[string]string + Entrypoint []string + Env map[string]string + HostConfigModifier func(*dockercontainer.HostConfig) + ExposedPorts []string + Cmd []string + Image string + Name string + Networks []string + WaitingFor wait.Strategy Address string Ports map[string]string @@ -50,15 +52,16 @@ func (c *Container) Start() error { req := testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - Mounts: testcontainers.Mounts(containerMounts...), - Entrypoint: c.Entrypoint, - Env: c.Env, - ExposedPorts: c.ExposedPorts, - Cmd: c.Cmd, - Image: c.Image, - Name: c.Name, - Networks: c.Networks, - WaitingFor: c.WaitingFor, + Mounts: testcontainers.Mounts(containerMounts...), + Entrypoint: c.Entrypoint, + Env: c.Env, + ExposedPorts: c.ExposedPorts, + HostConfigModifier: c.HostConfigModifier, + Cmd: c.Cmd, + Image: c.Image, + Name: c.Name, + Networks: c.Networks, + WaitingFor: c.WaitingFor, }, Started: true, }