From 89077c9df109dac8dbe6c979ebc181071c5e0db8 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Thu, 28 Apr 2022 11:45:48 +0100 Subject: [PATCH] Update. Signed-off-by: Bartlomiej Plotka --- test/e2e/compatibility_test.go | 106 +++++++++++++++++++-------------- test/e2e/e2ethanos/services.go | 17 +++--- test/e2e/rule_test.go | 6 +- test/e2e/rules_api_test.go | 6 +- 4 files changed, 72 insertions(+), 63 deletions(-) diff --git a/test/e2e/compatibility_test.go b/test/e2e/compatibility_test.go index 0eada75abd0..5012d2100d4 100644 --- a/test/e2e/compatibility_test.go +++ b/test/e2e/compatibility_test.go @@ -4,8 +4,8 @@ package e2e_test import ( + "fmt" "io/ioutil" - "net/http" "os" "path/filepath" "testing" @@ -21,11 +21,11 @@ import ( "github.com/thanos-io/thanos/test/e2e/e2ethanos" ) -// TestPromQLCompliance tests PromQL compatibility. +// TestPromQLCompliance tests PromQL compatibility against https://github.com/prometheus/compliance/tree/main/promql. // NOTE: This requires dockerization of compliance framework: https://github.com/prometheus/compliance/pull/46 // Test requires at least ~11m, so run this with `-test.timeout 9999m`. func TestPromQLCompliance(t *testing.T) { - //t.Skip("This is interactive test, it requires time to build up (scrape) the data. The data is also obtain from remote promlab servers.") + t.Skip("This is interactive test, it requires time to build up (scrape) the data. The data is also obtain from remote promlab servers.") e, err := e2e.NewDockerEnvironment("compatibility") testutil.Ok(t, err) @@ -77,24 +77,31 @@ scrape_configs: t.Run("receive", func(t *testing.T) { testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "receive.yaml"), - []byte(promLabelsPromQLConfig(prom, queryReceive, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm)) + []byte(promQLCompatConfig(prom, queryReceive, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm)) - stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml"))) + stdout, stderr, err := compliance.Exec(e2e.NewCommand( + "/promql-compliance-tester", + "-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml"), + "-config-file", "/promql-test-queries.yml", + )) t.Log(stdout, stderr) testutil.Ok(t, err) }) t.Run("sidecar", func(t *testing.T) { testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "sidecar.yaml"), - []byte(promLabelsPromQLConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm)) + []byte(promQLCompatConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm)) - stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml"))) + stdout, stderr, err := compliance.Exec(e2e.NewCommand( + "/promql-compliance-tester", + "-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml"), + "-config-file", "/promql-test-queries.yml", + )) t.Log(stdout, stderr) testutil.Ok(t, err) - }) } -func promLabelsPromQLConfig(reference *e2edb.Prometheus, target e2e.Runnable, dropLabels []string) string { +func promQLCompatConfig(reference *e2edb.Prometheus, target e2e.Runnable, dropLabels []string) string { return `reference_target_config: query_url: 'http://` + reference.InternalEndpoint("http") + `' @@ -113,9 +120,17 @@ query_tweaks: }() } -// TestAlertCompliance tests Alert compatibility against https://github.com/prometheus/compliance/blob/main/alert_generator/test-thanos.yaml. +type alwaysReadyProbe struct { +} + +func (p alwaysReadyProbe) Ready(e2e.Runnable) error { + return nil +} + +// TestAlertCompliance tests Alert compatibility against https://github.com/prometheus/compliance/blob/main/alert_generator. +// NOTE: This requires dockerization of compliance framework: https://github.com/prometheus/compliance/pull/46 func TestAlertCompliance_StatelessRuler(t *testing.T) { - t.Skip("This is an interactive test, mean to use with https://github.com/prometheus/compliance/tree/main/alert_generator") + t.Skip("This is an interactive test, using https://github.com/prometheus/compliance/tree/main/alert_generator testing is not optimized for CI runs") e, err := e2e.NewDockerEnvironment("alert_compatibility") testutil.Ok(t, err) @@ -125,10 +140,17 @@ func TestAlertCompliance_StatelessRuler(t *testing.T) { receive, err := e2ethanos.NewIngestingReceiver(e, "receive") testutil.Ok(t, err) query := e2edb.NewThanosQuerier(e, "query_receive", []string{receive.InternalEndpoint("grpc")}) - ruler, err := e2ethanos.NewStatelessRuler(e, "1", "rules", []alert.AlertmanagerConfig{ + + compliance := e.Runnable("alert_generator_compliance_tester").WithPorts(map[string]int{"http": 8080}).Init(e2e.StartOptions{ + Image: "alert_generator_compliance_tester:latest", + // Batch job with HTTP port, we will start it with Exec, no need for readiness. + Readiness: alwaysReadyProbe{}, + Command: e2e.NewCommandWithoutEntrypoint("tail", "-f", "/dev/null"), + }) + ruler := e2ethanos.NewStatelessRuler(e, "1", "rules", []alert.AlertmanagerConfig{ { EndpointsConfig: httpconfig.EndpointsConfig{ - StaticAddresses: []string{}, + StaticAddresses: []string{compliance.InternalEndpoint("http")}, Scheme: "http", }, Timeout: amTimeout, @@ -146,43 +168,35 @@ func TestAlertCompliance_StatelessRuler(t *testing.T) { }, []*config.RemoteWriteConfig{ {URL: &common_cfg.URL{URL: urlParse(t, e2ethanos.RemoteWriteEndpoint(receive.InternalEndpoint("remote-write")))}, Name: "thanos-receiver"}, }) - testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(receive, query, ruler)) + testutil.Ok(t, e2e.StartAndWaitReady(receive, query, ruler, compliance)) - // Pull fresh rules.yaml: + // Pull rules.yaml: { - resp, err := http.Get("https://raw.githubusercontent.com/prometheus/compliance/main/alert_generator/rules.yaml") - testutil.Ok(t, err) - testutil.Equals(t, http.StatusOK, resp.StatusCode) - b, err := ioutil.ReadAll(resp.Body) - testutil.Ok(t, err) - testutil.Ok(t, os.WriteFile(filepath.Join(ruler.Dir(), "rules", "rules.yaml"), b, os.ModePerm)) + stdout, stderr, err := compliance.Exec(e2e.NewCommand("cat", "/rules.yaml")) + testutil.Ok(t, err, stderr) + testutil.Ok(t, os.MkdirAll(filepath.Join(ruler.Dir(), "rules"), os.ModePerm)) + testutil.Ok(t, os.WriteFile(filepath.Join(ruler.Dir(), "rules", "rules.yaml"), []byte(stdout), os.ModePerm)) } - compliance := e.Runnable("promql-compliance-tester").Init(e2e.StartOptions{ - Image: "promql-compliance-tester:latest", - Command: e2e.NewCommandWithoutEntrypoint("tail", "-f", "/dev/null"), - }) - testutil.Ok(t, e2e.StartAndWaitReady(compliance)) + testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "test-thanos.yaml"), + []byte(alertCompatConfig(receive, query)), os.ModePerm)) - // Wait 10 minutes for Prometheus to scrape relevant data. - time.Sleep(10 * time.Minute) + fmt.Println(alertCompatConfig(receive, query)) + + stdout, stderr, err := compliance.Exec(e2e.NewCommand( + "/alert_generator_compliance_tester", "-config-file", filepath.Join(compliance.InternalDir(), "test-thanos.yaml")), + ) + t.Log(stdout, stderr) + testutil.Ok(t, err) +} + +func alertCompatConfig(receive e2e.Runnable, query e2e.Runnable) string { + return `settings: + remote_write_url: '` + e2ethanos.RemoteWriteEndpoint(receive.InternalEndpoint("http")) + `' + query_base_url: 'http://` + query.InternalEndpoint("http") + `' + rules_and_alerts_api_base_url: 'http://` + query.InternalEndpoint("http") + `' + alert_reception_server_port: 8080 + alert_message_parser: default +` - //t.Run("receive", func(t *testing.T) { - // testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "receive.yaml"), - // []byte(promLabelsPromQLConfig(prom, query, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm)) - // - // stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml"))) - // t.Log(stdout, stderr) - // testutil.Ok(t, err) - //}) - //t.Run("sidecar", func(t *testing.T) { - // testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "sidecar.yaml"), - // []byte(promLabelsPromQLConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm)) - // - // stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml"))) - // t.Log(stdout, stderr) - // testutil.Ok(t, err) - // - //}) } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 7a63d23ef14..c642db321ef 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -560,32 +560,32 @@ func NewIngestingReceiver(e e2e.Environment, name string) (e2e.InstrumentedRunna return receiver, nil } -func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config) (e2e.InstrumentedRunnable, error) { +func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config) e2e.InstrumentedRunnable { return newRuler(e, name, ruleSubDir, amCfg, queryCfg, nil) } -func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (e2e.InstrumentedRunnable, error) { +func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) e2e.InstrumentedRunnable { return newRuler(e, name, ruleSubDir, amCfg, queryCfg, remoteWriteCfg) } -func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (e2e.InstrumentedRunnable, error) { +func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) e2e.InstrumentedRunnable { dir := filepath.Join(e.SharedDir(), "data", "rule", name) container := filepath.Join(ContainerSharedDir, "data", "rule", name) if err := os.MkdirAll(dir, 0750); err != nil { - return nil, errors.Wrap(err, "create rule dir") + return e2e.NewErrInstrumentedRunnable(name, errors.Wrap(err, "create rule dir")) } amCfgBytes, err := yaml.Marshal(alert.AlertingConfig{ Alertmanagers: amCfg, }) if err != nil { - return nil, errors.Wrapf(err, "generate am file: %v", amCfg) + return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate am file: %v", amCfg)) } queryCfgBytes, err := yaml.Marshal(queryCfg) if err != nil { - return nil, errors.Wrapf(err, "generate query file: %v", queryCfg) + return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate query file: %v", queryCfg)) } ruleArgs := map[string]string{ @@ -609,12 +609,12 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"` }{remoteWriteCfg}) if err != nil { - return nil, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg) + return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg)) } ruleArgs["--remote-write.config"] = string(rwCfgBytes) } - ruler := NewService(e, + return NewService(e, fmt.Sprintf("rule-%v", name), DefaultImage(), e2e.NewCommand("rule", e2e.BuildArgs(ruleArgs)...), @@ -623,7 +623,6 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman 9091, ) - return ruler, nil } func NewAlertmanager(e e2e.Environment, name string) (e2e.InstrumentedRunnable, error) { diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 3ed1ff053c3..d100a07d78c 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -245,7 +245,7 @@ func TestRule(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(am1, am2)) - r, err := e2ethanos.NewTSDBRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ + r := e2ethanos.NewTSDBRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ { EndpointsConfig: httpconfig.EndpointsConfig{ FileSDConfigs: []httpconfig.FileSDConfig{ @@ -278,7 +278,6 @@ func TestRule(t *testing.T) { }, }, }) - testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r)) q, err := e2ethanos.NewQuerierBuilder(e, "1", r.InternalEndpoint("grpc")).Build() @@ -500,7 +499,7 @@ func TestRule_CanRemoteWriteData(t *testing.T) { q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc"), receiver2.InternalEndpoint("grpc")).Build() testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(q)) - r, err := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ + r := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ { EndpointsConfig: httpconfig.EndpointsConfig{ StaticAddresses: []string{ @@ -524,7 +523,6 @@ func TestRule_CanRemoteWriteData(t *testing.T) { {URL: &common_cfg.URL{URL: rwURL}, Name: "thanos-receiver"}, {URL: &common_cfg.URL{URL: rwURL2}, Name: "thanos-receiver2"}, }) - testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r)) // Wait until remote write samples are written to receivers successfully. diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 893ca214329..47db429b6ab 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -78,10 +78,8 @@ func TestRulesAPI_Fanout(t *testing.T) { } // Recreate rulers with the corresponding query config. - r1, err := e2ethanos.NewTSDBRuler(e, "rule1", thanosRulesSubDir, nil, queryCfg) - testutil.Ok(t, err) - r2, err := e2ethanos.NewTSDBRuler(e, "rule2", thanosRulesSubDir, nil, queryCfg) - testutil.Ok(t, err) + r1 := e2ethanos.NewTSDBRuler(e, "rule1", thanosRulesSubDir, nil, queryCfg) + r2 := e2ethanos.NewTSDBRuler(e, "rule2", thanosRulesSubDir, nil, queryCfg) testutil.Ok(t, e2e.StartAndWaitReady(r1, r2)) stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc")}