Skip to content

Commit

Permalink
Update.
Browse files Browse the repository at this point in the history 
Signed-off-by: Bartlomiej Plotka <[email protected]>
  • Loading branch information
@bwplotka
bwplotka committed Apr 28, 2022
1 parent 8fc03b9 commit 89077c9
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 63 deletions.
106 changes: 60 additions & 46 deletions test/e2e/compatibility_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
package e2e_test

import (
"fmt"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"testing"
Expand All @@ -21,11 +21,11 @@ import (
"github.com/thanos-io/thanos/test/e2e/e2ethanos"
)

// TestPromQLCompliance tests PromQL compatibility.
// TestPromQLCompliance tests PromQL compatibility against https://github.com/prometheus/compliance/tree/main/promql.
// NOTE: This requires dockerization of compliance framework: https://github.com/prometheus/compliance/pull/46
// Test requires at least ~11m, so run this with `-test.timeout 9999m`.
func TestPromQLCompliance(t *testing.T) {
//t.Skip("This is interactive test, it requires time to build up (scrape) the data. The data is also obtain from remote promlab servers.")
t.Skip("This is interactive test, it requires time to build up (scrape) the data. The data is also obtain from remote promlab servers.")

e, err := e2e.NewDockerEnvironment("compatibility")
testutil.Ok(t, err)
Expand Down Expand Up @@ -77,24 +77,31 @@ scrape_configs:

t.Run("receive", func(t *testing.T) {
testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "receive.yaml"),
[]byte(promLabelsPromQLConfig(prom, queryReceive, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm))
[]byte(promQLCompatConfig(prom, queryReceive, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm))

stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml")))
stdout, stderr, err := compliance.Exec(e2e.NewCommand(
"/promql-compliance-tester",
"-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml"),
"-config-file", "/promql-test-queries.yml",
))
t.Log(stdout, stderr)
testutil.Ok(t, err)
})
t.Run("sidecar", func(t *testing.T) {
testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "sidecar.yaml"),
[]byte(promLabelsPromQLConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm))
[]byte(promQLCompatConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm))

stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml")))
stdout, stderr, err := compliance.Exec(e2e.NewCommand(
"/promql-compliance-tester",
"-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml"),
"-config-file", "/promql-test-queries.yml",
))
t.Log(stdout, stderr)
testutil.Ok(t, err)

})
}

func promLabelsPromQLConfig(reference *e2edb.Prometheus, target e2e.Runnable, dropLabels []string) string {
func promQLCompatConfig(reference *e2edb.Prometheus, target e2e.Runnable, dropLabels []string) string {
return `reference_target_config:
query_url: 'http://` + reference.InternalEndpoint("http") + `'
Expand All @@ -113,9 +120,17 @@ query_tweaks:
}()
}

// TestAlertCompliance tests Alert compatibility against https://github.com/prometheus/compliance/blob/main/alert_generator/test-thanos.yaml.
type alwaysReadyProbe struct {
}

func (p alwaysReadyProbe) Ready(e2e.Runnable) error {
return nil
}

// TestAlertCompliance tests Alert compatibility against https://github.com/prometheus/compliance/blob/main/alert_generator.
// NOTE: This requires dockerization of compliance framework: https://github.com/prometheus/compliance/pull/46
func TestAlertCompliance_StatelessRuler(t *testing.T) {
t.Skip("This is an interactive test, mean to use with https://github.com/prometheus/compliance/tree/main/alert_generator")
t.Skip("This is an interactive test, using https://github.com/prometheus/compliance/tree/main/alert_generator testing is not optimized for CI runs")

e, err := e2e.NewDockerEnvironment("alert_compatibility")
testutil.Ok(t, err)
Expand All @@ -125,10 +140,17 @@ func TestAlertCompliance_StatelessRuler(t *testing.T) {
receive, err := e2ethanos.NewIngestingReceiver(e, "receive")
testutil.Ok(t, err)
query := e2edb.NewThanosQuerier(e, "query_receive", []string{receive.InternalEndpoint("grpc")})
ruler, err := e2ethanos.NewStatelessRuler(e, "1", "rules", []alert.AlertmanagerConfig{

compliance := e.Runnable("alert_generator_compliance_tester").WithPorts(map[string]int{"http": 8080}).Init(e2e.StartOptions{
Image: "alert_generator_compliance_tester:latest",
// Batch job with HTTP port, we will start it with Exec, no need for readiness.
Readiness: alwaysReadyProbe{},
Command: e2e.NewCommandWithoutEntrypoint("tail", "-f", "/dev/null"),
})
ruler := e2ethanos.NewStatelessRuler(e, "1", "rules", []alert.AlertmanagerConfig{
{
EndpointsConfig: httpconfig.EndpointsConfig{
StaticAddresses: []string{},
StaticAddresses: []string{compliance.InternalEndpoint("http")},
Scheme: "http",
},
Timeout: amTimeout,
Expand All @@ -146,43 +168,35 @@ func TestAlertCompliance_StatelessRuler(t *testing.T) {
}, []*config.RemoteWriteConfig{
{URL: &common_cfg.URL{URL: urlParse(t, e2ethanos.RemoteWriteEndpoint(receive.InternalEndpoint("remote-write")))}, Name: "thanos-receiver"},
})
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(receive, query, ruler))
testutil.Ok(t, e2e.StartAndWaitReady(receive, query, ruler, compliance))

// Pull fresh rules.yaml:
// Pull rules.yaml:
{
resp, err := http.Get("https://raw.githubusercontent.com/prometheus/compliance/main/alert_generator/rules.yaml")
testutil.Ok(t, err)
testutil.Equals(t, http.StatusOK, resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
testutil.Ok(t, err)
testutil.Ok(t, os.WriteFile(filepath.Join(ruler.Dir(), "rules", "rules.yaml"), b, os.ModePerm))
stdout, stderr, err := compliance.Exec(e2e.NewCommand("cat", "/rules.yaml"))
testutil.Ok(t, err, stderr)
testutil.Ok(t, os.MkdirAll(filepath.Join(ruler.Dir(), "rules"), os.ModePerm))
testutil.Ok(t, os.WriteFile(filepath.Join(ruler.Dir(), "rules", "rules.yaml"), []byte(stdout), os.ModePerm))
}

compliance := e.Runnable("promql-compliance-tester").Init(e2e.StartOptions{
Image: "promql-compliance-tester:latest",
Command: e2e.NewCommandWithoutEntrypoint("tail", "-f", "/dev/null"),
})
testutil.Ok(t, e2e.StartAndWaitReady(compliance))
testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "test-thanos.yaml"),
[]byte(alertCompatConfig(receive, query)), os.ModePerm))

// Wait 10 minutes for Prometheus to scrape relevant data.
time.Sleep(10 * time.Minute)
fmt.Println(alertCompatConfig(receive, query))

stdout, stderr, err := compliance.Exec(e2e.NewCommand(
"/alert_generator_compliance_tester", "-config-file", filepath.Join(compliance.InternalDir(), "test-thanos.yaml")),
)
t.Log(stdout, stderr)
testutil.Ok(t, err)
}

func alertCompatConfig(receive e2e.Runnable, query e2e.Runnable) string {
return `settings:
remote_write_url: '` + e2ethanos.RemoteWriteEndpoint(receive.InternalEndpoint("http")) + `'
query_base_url: 'http://` + query.InternalEndpoint("http") + `'
rules_and_alerts_api_base_url: 'http://` + query.InternalEndpoint("http") + `'
alert_reception_server_port: 8080
alert_message_parser: default
`

//t.Run("receive", func(t *testing.T) {
// testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "receive.yaml"),
// []byte(promLabelsPromQLConfig(prom, query, []string{"prometheus", "receive", "tenant_id"})), os.ModePerm))
//
// stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "receive.yaml")))
// t.Log(stdout, stderr)
// testutil.Ok(t, err)
//})
//t.Run("sidecar", func(t *testing.T) {
// testutil.Ok(t, ioutil.WriteFile(filepath.Join(compliance.Dir(), "sidecar.yaml"),
// []byte(promLabelsPromQLConfig(prom, querySidecar, []string{"prometheus"})), os.ModePerm))
//
// stdout, stderr, err := compliance.Exec(e2e.NewCommand("/promql-compliance-tester", "-config-file", filepath.Join(compliance.InternalDir(), "sidecar.yaml")))
// t.Log(stdout, stderr)
// testutil.Ok(t, err)
//
//})
}
17 changes: 8 additions & 9 deletions test/e2e/e2ethanos/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,32 +560,32 @@ func NewIngestingReceiver(e e2e.Environment, name string) (e2e.InstrumentedRunna
return receiver, nil
}

func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config) (e2e.InstrumentedRunnable, error) {
func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config) e2e.InstrumentedRunnable {
return newRuler(e, name, ruleSubDir, amCfg, queryCfg, nil)
}

func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (e2e.InstrumentedRunnable, error) {
func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) e2e.InstrumentedRunnable {
return newRuler(e, name, ruleSubDir, amCfg, queryCfg, remoteWriteCfg)
}

func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (e2e.InstrumentedRunnable, error) {
func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) e2e.InstrumentedRunnable {
dir := filepath.Join(e.SharedDir(), "data", "rule", name)
container := filepath.Join(ContainerSharedDir, "data", "rule", name)

if err := os.MkdirAll(dir, 0750); err != nil {
return nil, errors.Wrap(err, "create rule dir")
return e2e.NewErrInstrumentedRunnable(name, errors.Wrap(err, "create rule dir"))
}

amCfgBytes, err := yaml.Marshal(alert.AlertingConfig{
Alertmanagers: amCfg,
})
if err != nil {
return nil, errors.Wrapf(err, "generate am file: %v", amCfg)
return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate am file: %v", amCfg))
}

queryCfgBytes, err := yaml.Marshal(queryCfg)
if err != nil {
return nil, errors.Wrapf(err, "generate query file: %v", queryCfg)
return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate query file: %v", queryCfg))
}

ruleArgs := map[string]string{
Expand All @@ -609,12 +609,12 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman
RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"`
}{remoteWriteCfg})
if err != nil {
return nil, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg)
return e2e.NewErrInstrumentedRunnable(name, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg))
}
ruleArgs["--remote-write.config"] = string(rwCfgBytes)
}

ruler := NewService(e,
return NewService(e,
fmt.Sprintf("rule-%v", name),
DefaultImage(),
e2e.NewCommand("rule", e2e.BuildArgs(ruleArgs)...),
Expand All @@ -623,7 +623,6 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman
9091,
)

return ruler, nil
}

func NewAlertmanager(e e2e.Environment, name string) (e2e.InstrumentedRunnable, error) {
Expand Down
6 changes: 2 additions & 4 deletions test/e2e/rule_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func TestRule(t *testing.T) {
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(am1, am2))

r, err := e2ethanos.NewTSDBRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{
r := e2ethanos.NewTSDBRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{
{
EndpointsConfig: httpconfig.EndpointsConfig{
FileSDConfigs: []httpconfig.FileSDConfig{
Expand Down Expand Up @@ -278,7 +278,6 @@ func TestRule(t *testing.T) {
},
},
})
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(r))

q, err := e2ethanos.NewQuerierBuilder(e, "1", r.InternalEndpoint("grpc")).Build()
Expand Down Expand Up @@ -500,7 +499,7 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc"), receiver2.InternalEndpoint("grpc")).Build()
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(q))
r, err := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{
r := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{
{
EndpointsConfig: httpconfig.EndpointsConfig{
StaticAddresses: []string{
Expand All @@ -524,7 +523,6 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
{URL: &common_cfg.URL{URL: rwURL}, Name: "thanos-receiver"},
{URL: &common_cfg.URL{URL: rwURL2}, Name: "thanos-receiver2"},
})
testutil.Ok(t, err)
testutil.Ok(t, e2e.StartAndWaitReady(r))

// Wait until remote write samples are written to receivers successfully.
Expand Down
6 changes: 2 additions & 4 deletions test/e2e/rules_api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,8 @@ func TestRulesAPI_Fanout(t *testing.T) {
}

// Recreate rulers with the corresponding query config.
r1, err := e2ethanos.NewTSDBRuler(e, "rule1", thanosRulesSubDir, nil, queryCfg)
testutil.Ok(t, err)
r2, err := e2ethanos.NewTSDBRuler(e, "rule2", thanosRulesSubDir, nil, queryCfg)
testutil.Ok(t, err)
r1 := e2ethanos.NewTSDBRuler(e, "rule1", thanosRulesSubDir, nil, queryCfg)
r2 := e2ethanos.NewTSDBRuler(e, "rule2", thanosRulesSubDir, nil, queryCfg)
testutil.Ok(t, e2e.StartAndWaitReady(r1, r2))

stores := []string{sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc"), r1.InternalEndpoint("grpc"), r2.InternalEndpoint("grpc")}
Expand Down

0 comments on commit 89077c9

Please sign in to comment.