Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(inputs.vault): Use http client to handle redirects correctly #14153

Merged
merged 5 commits into from
Oct 23, 2023
Merged

fix(inputs.vault): Use http client to handle redirects correctly #14153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ad87628
Add integration test
srebhan Oct 20, 2023
17db21d
Add test for temporary redirect (issue #14138)
srebhan Oct 20, 2023
96a0f29
Make response-header-timeout available for http client
srebhan Oct 20, 2023
5df513e
Switch to http-client usage
srebhan Oct 20, 2023
156ba68
Make linter happy
srebhan Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions plugins/common/http/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ import (

// Common HTTP client struct.
type HTTPClientConfig struct {
Timeout config.Duration `toml:"timeout"`
IdleConnTimeout config.Duration `toml:"idle_conn_timeout"`
MaxIdleConns int `toml:"max_idle_conn"`
MaxIdleConnsPerHost int `toml:"max_idle_conn_per_host"`
Timeout config.Duration `toml:"timeout"`
IdleConnTimeout config.Duration `toml:"idle_conn_timeout"`
MaxIdleConns int `toml:"max_idle_conn"`
MaxIdleConnsPerHost int `toml:"max_idle_conn_per_host"`
ResponseHeaderTimeout config.Duration `toml:"response_timeout"`

proxy.HTTPProxy
tls.ClientConfig
Expand All @@ -42,11 +43,12 @@ func (h *HTTPClientConfig) CreateClient(ctx context.Context, log telegraf.Logger
}

transport := &http.Transport{
TLSClientConfig: tlsCfg,
Proxy: prox,
IdleConnTimeout: time.Duration(h.IdleConnTimeout),
MaxIdleConns: h.MaxIdleConns,
MaxIdleConnsPerHost: h.MaxIdleConnsPerHost,
TLSClientConfig: tlsCfg,
Proxy: prox,
IdleConnTimeout: time.Duration(h.IdleConnTimeout),
MaxIdleConns: h.MaxIdleConns,
MaxIdleConnsPerHost: h.MaxIdleConnsPerHost,
ResponseHeaderTimeout: time.Duration(h.ResponseHeaderTimeout),
}

// Register "http+unix" and "https+unix" protocol handler.
Expand Down
51 changes: 51 additions & 0 deletions plugins/inputs/vault/testdata/response_integration_1.13.3.influx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
vault.audit.log_request_failure count=3i,max=0i,mean=0,min=0i,rate=0,stddev=0,sum=0i 1697798140000000000
vault.audit.log_response_failure count=3i,max=0i,mean=0,min=0i,rate=0,stddev=0,sum=0i 1697798140000000000
vault.barrier.estimated_encryptions,term=1 count=33i,max=1i,mean=1,min=1i,rate=3.3,stddev=0,sum=33i 1697798140000000000
vault.cache.hit count=15i,max=1i,mean=1,min=1i,rate=1.5,stddev=0,sum=15i 1697798140000000000
vault.cache.miss count=54i,max=1i,mean=1,min=1i,rate=5.4,stddev=0,sum=54i 1697798140000000000
vault.cache.write count=28i,max=1i,mean=1,min=1i,rate=2.8,stddev=0,sum=28i 1697798140000000000
vault.seal.decrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000
vault.seal.encrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000
vault.seal.shamir.decrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000
vault.seal.shamir.encrypt count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000
vault.token.create_root count=2i,max=1i,mean=1,min=1i,rate=0.2,stddev=0,sum=2i 1697798140000000000
vault.token.creation,auth_method=token,cluster=vault-cluster-57e15bbb,creation_ttl=1m,mount_point=auth/token/,namespace=root,token_type=service count=1i,max=1i,mean=1,min=1i,rate=0.1,stddev=0,sum=1i 1697798140000000000
vault.core.locked_users value=0i 1697798140000000000
vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=false,type=auth value=1i 1697798140000000000
vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=true,type=auth value=0i 1697798140000000000
vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=false,type=logical value=3i 1697798140000000000
vault.core.mount_table.num_entries,cluster=vault-cluster-57e15bbb,local=true,type=logical value=1i 1697798140000000000
vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=false,type=auth value=251i 1697798140000000000
vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=true,type=auth value=56i 1697798140000000000
vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=false,type=logical value=556i 1697798140000000000
vault.core.mount_table.size,cluster=vault-cluster-57e15bbb,local=true,type=logical value=302i 1697798140000000000
vault.audit.log_request count=3i,max=0.00892999954521656,mean=0.005283333205928405,min=0.0029299999587237835,rate=0.0015849999617785215,stddev=0.0032022697614016125,sum=0.015849999617785215 1697798140000000000
vault.audit.log_response count=3i,max=0.006140000186860561,mean=0.004420000050837795,min=0.00343999988399446,rate=0.0013260000152513385,stddev=0.0014943896328128183,sum=0.013260000152513385 1697798140000000000
vault.barrier.delete count=4i,max=0.004600000102072954,mean=0.003989999939221889,min=0.0035099999513477087,rate=0.0015959999756887556,stddev=0.00045658157079035244,sum=0.015959999756887555 1697798140000000000
vault.barrier.get count=75i,max=0.049591001123189926,mean=0.00808781327912584,min=0.0007999999797903001,rate=0.060658599593443795,stddev=0.010314610354373298,sum=0.606585995934438 1697798140000000000
vault.barrier.list count=37i,max=0.0049310000613331795,mean=0.0013254594665045875,min=0.0002800000074785203,rate=0.004904200026066974,stddev=0.0010832305178496164,sum=0.04904200026066974 1697798140000000000
vault.barrier.put count=31i,max=0.05448000133037567,mean=0.013786516182364957,min=0.0061900001019239426,rate=0.042738200165331364,stddev=0.008650664562076636,sum=0.42738200165331364 1697798140000000000
vault.core.check_token count=3i,max=1.133538007736206,mean=0.4793433391799529,min=0.02119000069797039,rate=0.1438030017539859,stddev=0.5815098232303996,sum=1.4380300175398588 1697798140000000000
vault.core.fetch_acl_and_token count=3i,max=0.010940000414848328,mean=0.00783000017205874,min=0.006060000043362379,rate=0.0023490000516176225,stddev=0.0027019069443178717,sum=0.023490000516176224 1697798140000000000
vault.core.handle_request count=3i,max=1.790362000465393,mean=1.1120976607004802,min=0.525954008102417,rate=0.33362929821014403,stddev=0.6372178266032051,sum=3.3362929821014404 1697798140000000000
vault.core.post_unseal count=2i,max=171.49169921875,mean=86.97485756874084,min=2.4580159187316895,rate=17.394971513748168,stddev=119.52486371038222,sum=173.9497151374817 1697798140000000000
vault.core.pre_seal count=1i,max=0.46343299746513367,mean=0.46343299746513367,min=0.46343299746513367,rate=0.04634329974651337,stddev=0,sum=0.46343299746513367 1697798140000000000
vault.expire.register-auth count=1i,max=0.2956019937992096,mean=0.2956019937992096,min=0.2956019937992096,rate=0.029560199379920958,stddev=0,sum=0.2956019937992096 1697798140000000000
vault.expire.revoke count=1i,max=0.39290300011634827,mean=0.39290300011634827,min=0.39290300011634827,rate=0.039290300011634825,stddev=0,sum=0.39290300011634827 1697798140000000000
vault.expire.revoke-by-token count=1i,max=0.14829100668430328,mean=0.14829100668430328,min=0.14829100668430328,rate=0.014829100668430328,stddev=0,sum=0.14829100668430328 1697798140000000000
vault.expire.revoke-common count=2i,max=0.3899630010128021,mean=0.25605200231075287,min=0.12214100360870361,rate=0.05121040046215057,stddev=0.18937875051536399,sum=0.5121040046215057 1697798140000000000
vault.policy.get_policy count=11i,max=0.46198299527168274,mean=0.056309545021080834,min=0.001560000004246831,rate=0.06194049952318892,stddev=0.13695683947079185,sum=0.6194049952318892 1697798140000000000
vault.route.update.auth-token- count=2i,max=0.4904730021953583,mean=0.3715669959783554,min=0.25266098976135254,rate=0.07431339919567108,stddev=0.168158486639705,sum=0.7431339919567108 1697798140000000000
vault.route.update.sys- count=1i,max=0.6211339831352234,mean=0.6211339831352234,min=0.6211339831352234,rate=0.06211339831352234,stddev=0,sum=0.6211339831352234 1697798140000000000
vault.seal.decrypt.time count=1i,max=0.019300000742077827,mean=0.019300000742077827,min=0.019300000742077827,rate=0.0019300000742077828,stddev=0,sum=0.019300000742077827 1697798140000000000
vault.seal.encrypt.time count=1i,max=0.008750000037252903,mean=0.008750000037252903,min=0.008750000037252903,rate=0.0008750000037252903,stddev=0,sum=0.008750000037252903 1697798140000000000
vault.seal.shamir.decrypt.time count=1i,max=0.04343099892139435,mean=0.04343099892139435,min=0.04343099892139435,rate=0.004343099892139435,stddev=0,sum=0.04343099892139435 1697798140000000000
vault.seal.shamir.encrypt.time count=1i,max=0.25516200065612793,mean=0.25516200065612793,min=0.25516200065612793,rate=0.025516200065612792,stddev=0,sum=0.25516200065612793 1697798140000000000
vault.token.create count=2i,max=0.7710049748420715,mean=0.42180248722434044,min=0.07259999960660934,rate=0.0843604974448681,stddev=0.4938468940034181,sum=0.8436049744486809 1697798140000000000
vault.token.createAccessor count=2i,max=0.032510001212358475,mean=0.02841000072658062,min=0.024310000240802765,rate=0.005682000145316124,stddev=0.00579827629272332,sum=0.05682000145316124 1697798140000000000
vault.token.lookup count=6i,max=0.08506099879741669,mean=0.04382349985341231,min=0.027969999238848686,rate=0.026294099912047387,stddev=0.021177076886439525,sum=0.26294099912047386 1697798140000000000
vault.token.revoke-tree count=1i,max=0.2888520061969757,mean=0.2888520061969757,min=0.2888520061969757,rate=0.02888520061969757,stddev=0,sum=0.2888520061969757 1697798140000000000
vault.token.store count=1i,max=0.023420000448822975,mean=0.023420000448822975,min=0.023420000448822975,rate=0.0023420000448822974,stddev=0,sum=0.023420000448822975 1697798140000000000
vault.core.unseal count=1i,max=2.752518892288208,mean=2.752518892288208,min=2.752518892288208,rate=0.2752518892288208,stddev=0,sum=2.752518892288208 1697798140000000000
vault.core.unsealed,cluster=a value=0i 1697798140000000000
vault.core.unsealed,cluster=vault-cluster-57e15bbb value=1i 1697798140000000000
50 changes: 23 additions & 27 deletions plugins/inputs/vault/vault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
package vault

import (
"context"
_ "embed"
"encoding/json"
"fmt"
Expand All @@ -13,7 +14,7 @@ import (
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/common/tls"
httpcommon "github.com/influxdata/telegraf/plugins/common/http"
"github.com/influxdata/telegraf/plugins/inputs"
)

Expand All @@ -22,28 +23,17 @@ var sampleConfig string

// Vault configuration object
type Vault struct {
URL string `toml:"url"`
URL string `toml:"url"`
TokenFile string `toml:"token_file"`
Token string `toml:"token"`
Log telegraf.Logger `toml:"-"`
httpcommon.HTTPClientConfig

TokenFile string `toml:"token_file"`
Token string `toml:"token"`

ResponseTimeout config.Duration `toml:"response_timeout"`

tls.ClientConfig

roundTripper http.RoundTripper
client *http.Client
}

const timeLayout = "2006-01-02 15:04:05 -0700 MST"

func init() {
inputs.Add("vault", func() telegraf.Input {
return &Vault{
ResponseTimeout: config.Duration(5 * time.Second),
}
})
}

func (*Vault) SampleConfig() string {
return sampleConfig
}
Expand All @@ -69,16 +59,12 @@ func (n *Vault) Init() error {
n.Token = strings.TrimSpace(string(token))
}

tlsCfg, err := n.ClientConfig.TLSConfig()
ctx := context.Background()
client, err := n.HTTPClientConfig.CreateClient(ctx, n.Log)
if err != nil {
return fmt.Errorf("setting up TLS configuration failed: %w", err)
}

n.roundTripper = &http.Transport{
TLSHandshakeTimeout: 5 * time.Second,
TLSClientConfig: tlsCfg,
ResponseHeaderTimeout: time.Duration(n.ResponseTimeout),
return fmt.Errorf("creating client failed: %w", err)
}
n.client = client

return nil
}
Expand All @@ -102,7 +88,7 @@ func (n *Vault) loadJSON(url string) (*SysMetrics, error) {
req.Header.Set("X-Vault-Token", n.Token)
req.Header.Add("Accept", "application/json")

resp, err := n.roundTripper.RoundTrip(req)
resp, err := n.client.Do(req)
if err != nil {
return nil, fmt.Errorf("error making HTTP request to %q: %w", url, err)
}
Expand Down Expand Up @@ -191,3 +177,13 @@ func buildVaultMetrics(acc telegraf.Accumulator, sysMetrics *SysMetrics) error {

return nil
}

func init() {
inputs.Add("vault", func() telegraf.Input {
return &Vault{
HTTPClientConfig: httpcommon.HTTPClientConfig{
ResponseHeaderTimeout: config.Duration(5 * time.Second),
},
}
})
}
147 changes: 147 additions & 0 deletions plugins/inputs/vault/vault_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,20 @@ import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"

dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
"github.com/google/go-cmp/cmp"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
"github.com/stretchr/testify/require"
"github.com/testcontainers/testcontainers-go/wait"
)

func TestVaultStats(t *testing.T) {
Expand Down Expand Up @@ -96,3 +104,142 @@ func TestVaultStats(t *testing.T) {
})
}
}

func TestRedirect(t *testing.T) {
expected := []telegraf.Metric{
testutil.MustMetric(
"vault.raft.replication.appendEntries.logs",
map[string]string{
"peer_id": "clustnode-02",
},
map[string]interface{}{
"count": int(130),
"rate": float64(0.2),
"sum": int(2),
"min": int(0),
"max": int(1),
"mean": float64(0.015384615384615385),
"stddev": float64(0.12355304447984486),
},
time.Unix(1638287340, 0),
1,
),
testutil.MustMetric(
"vault.core.unsealed",
map[string]string{
"cluster": "vault-cluster-23b671c7",
},
map[string]interface{}{
"value": int(1),
},
time.Unix(1638287340, 0),
2,
),
testutil.MustMetric(
"vault.token.lookup",
map[string]string{},
map[string]interface{}{
"count": int(5135),
"max": float64(16.22449493408203),
"mean": float64(0.1698389152269865),
"min": float64(0.06690400093793869),
"rate": float64(87.21228296905755),
"stddev": float64(0.24637634000854705),
"sum": float64(872.1228296905756),
},
time.Unix(1638287340, 0),
1,
),
}

response, err := os.ReadFile("testdata/response_key_metrics.json")
require.NoError(t, err)

server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/v1/sys/metrics":
redirectURL := "http://" + r.Host + "/custom/metrics"
http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect)
case "/custom/metrics":
w.WriteHeader(http.StatusOK)
_, _ = w.Write(response)
}
}))
defer server.Close()

// Setup the plugin
plugin := &Vault{
URL: server.URL,
Token: "s.CDDrgg5zPv5ssI0Z2P4qxJj2",
}
require.NoError(t, plugin.Init())

var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))
actual := acc.GetTelegrafMetrics()
testutil.RequireMetricsEqual(t, expected, actual)
}

func TestIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}

// Start the docker container
container := testutil.Container{
Image: "vault:1.13.3",
ExposedPorts: []string{"8200"},
Env: map[string]string{
"VAULT_DEV_ROOT_TOKEN_ID": "root",
},
HostConfigModifier: func(hc *dockercontainer.HostConfig) {
hc.CapAdd = []string{"IPC_LOCK"}
},
WaitingFor: wait.ForAll(
wait.ForLog("Root Token: root"),
wait.ForListeningPort(nat.Port("8200")),
),
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()

// Setup the plugin
port := container.Ports["8200"]
plugin := &Vault{
URL: "http://" + container.Address + ":" + port,
Token: "root",
}
require.NoError(t, plugin.Init())

// Setup the expectations
buf, err := os.ReadFile(filepath.Join("testdata", "response_integration_1.13.3.influx"))
require.NoError(t, err)
parser := &influx.Parser{}
require.NoError(t, parser.Init())
raw, err := parser.Parse(buf)
require.NoError(t, err)
expected := make([]telegraf.Metric, 0, len(raw))
for _, r := range raw {
vt := telegraf.Counter
switch r.Name() {
case "vault.core.locked_users", "vault.core.mount_table.num_entries",
"vault.core.mount_table.size", "vault.core.unsealed":
vt = telegraf.Gauge
}
m := metric.New(r.Name(), r.Tags(), r.Fields(), r.Time(), vt)
expected = append(expected, m)
}

options := []cmp.Option{
testutil.SortMetrics(),
testutil.IgnoreTags("cluster"),
testutil.IgnoreTime(),
}

// Collect the metrics and compare
var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))

actual := acc.GetTelegrafMetrics()
testutil.RequireMetricsStructureEqual(t, expected, actual, options...)
}
Loading
Loading