update e2e

Signed-off-by: Ben Ye <[email protected]>
thanos-io · Oct 31, 2022 · d865211 · d865211
1 parent fe8f8dc
commit d865211
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 12 deletions.
diff --git a/docs/components/rule.md b/docs/components/rule.md
@@ -391,8 +391,8 @@ Flags:
                                  https://thanos.io/tip/components/rule.md/#configuration.
                                  If defined, it takes precedence over the
                                  '--query' and '--query.sd-files' flags.
-      --query.default-step=1s    Default range query step to use. This is only
-                                 used in stateless Ruler and alert state
+      --query.default-step=1s    Default range query step to use. This is
+                                 only used in stateless Ruler and alert state
                                  restoration.
       --query.http-method=POST   HTTP method to use when sending queries.
                                  Possible options: [GET, POST]

diff --git a/pkg/rules/queryable.go b/pkg/rules/queryable.go
@@ -10,7 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cortexproject/cortex/pkg/querier/series"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -19,6 +18,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/storage"
 
+	"github.com/thanos-io/thanos/internal/cortex/querier/series"
 	"github.com/thanos-io/thanos/pkg/httpconfig"
 	"github.com/thanos-io/thanos/pkg/promclient"
 	"github.com/thanos-io/thanos/pkg/store/storepb"

diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go
@@ -605,11 +605,11 @@ func TestRule_CanRemoteWriteData(t *testing.T) {
 func TestStatelessRulerAlertStateRestore(t *testing.T) {
 	t.Parallel()
 
-	e, err := e2e.NewDockerEnvironment("e2e_test_stateless_rule_alert_state_restore")
+	e, err := e2e.NewDockerEnvironment("stateless-state")
 	testutil.Ok(t, err)
 	t.Cleanup(e2ethanos.CleanScenario(t, e))
 
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	t.Cleanup(cancel)
 
 	am := e2ethanos.NewAlertmanager(e, "1")
@@ -623,7 +623,7 @@ func TestStatelessRulerAlertStateRestore(t *testing.T) {
 		WithReplicaLabels("replica", "receive").Init()
 	testutil.Ok(t, e2e.StartAndWaitReady(q))
 	rulesSubDir := "rules"
-	rulers := []e2e.InstrumentedRunnable{}
+	var rulers []*e2emon.InstrumentedRunnable
 	for i := 1; i <= 2; i++ {
 		rFuture := e2ethanos.NewRulerBuilder(e, fmt.Sprintf("%d", i))
 		rulesPath := filepath.Join(rFuture.Dir(), rulesSubDir)
@@ -662,13 +662,24 @@ func TestStatelessRulerAlertStateRestore(t *testing.T) {
 	// Start the ruler 1 first.
 	testutil.Ok(t, e2e.StartAndWaitReady(rulers[0]))
 
-	// Wait until remote write samples are written to receivers successfully.
-	testutil.Ok(t, rulers[0].WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"prometheus_remote_storage_samples_total"}, e2e.WaitMissingMetrics()))
+	// Wait until the alert firing and ALERTS_FOR_STATE
+	// series has been written to receiver successfully.
+	queryAndAssertSeries(t, ctx, q.Endpoint("http"), func() string {
+		return "ALERTS_FOR_STATE"
+	}, time.Now, promclient.QueryOptions{
+		Deduplicate: true,
+	}, []model.Metric{
+		{
+			"__name__":  "ALERTS_FOR_STATE",
+			"alertname": "TestAlert_RuleHoldDuration",
+			"severity":  "page",
+			"tenant_id": "default-tenant",
+		},
+	})
 
-	// Wait until the alert firing.
 	var alerts []*rulespb.AlertInstance
 	client := promclient.NewDefaultClient()
-	err = runutil.Repeat(time.Second*1, ctx.Done(), func() error {
+	err = runutil.Retry(time.Second*1, ctx.Done(), func() error {
 		alerts, err = client.AlertsInGRPC(ctx, urlParse(t, "http://"+rulers[0].Endpoint("http")))
 		testutil.Ok(t, err)
 		if len(alerts) > 0 {
@@ -688,10 +699,10 @@ func TestStatelessRulerAlertStateRestore(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(rulers[1]))
 
 	// Wait for 4 rule evaluation iterations to make sure the alert state is restored.
-	testutil.Ok(t, rulers[1].WaitSumMetricsWithOptions(e2e.GreaterOrEqual(4), []string{"prometheus_rule_group_duration_seconds_count"}, e2e.WaitMissingMetrics()))
+	testutil.Ok(t, rulers[1].WaitSumMetricsWithOptions(e2emon.GreaterOrEqual(4), []string{"prometheus_rule_group_iterations_total"}, e2emon.WaitMissingMetrics()))
 
 	// Wait until the alert is firing on the second ruler.
-	err = runutil.Repeat(time.Second*1, ctx.Done(), func() error {
+	err = runutil.Retry(time.Second*1, ctx.Done(), func() error {
 		alerts, err = client.AlertsInGRPC(ctx, urlParse(t, "http://"+rulers[1].Endpoint("http")))
 		testutil.Ok(t, err)
 		if len(alerts) > 0 {