Skip to content

Commit

Permalink
draft: refactored compact progress test
Browse files Browse the repository at this point in the history 
Signed-off-by: metonymic-smokey <[email protected]>
  • Loading branch information
@metonymic-smokey
metonymic-smokey committed Nov 5, 2021
1 parent a58b63f commit fe3355e
Showing 1 changed file with 40 additions and 231 deletions.
271 changes: 40 additions & 231 deletions pkg/compact/compact_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,21 @@ func BenchmarkGatherNoCompactionMarkFilter_Filter(b *testing.B) {

}

func createBlockMeta(id uint64, minTime, maxTime int64, labels map[string]string, resolution int64) *metadata.Meta {
m := &metadata.Meta{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(id, nil),
MinTime: int64(time.Duration(minTime) * time.Hour / time.Millisecond),
MaxTime: int64(time.Duration(maxTime) * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Labels: labels,
},
}

return m
}

func TestCompactProgressCalculate(t *testing.T) {
type planResult struct {
compactionBlocks, compactionRuns float64
Expand Down Expand Up @@ -219,97 +234,14 @@ func TestCompactProgressCalculate(t *testing.T) {
{
testName: "first_test",
input: []*metadata.Meta{
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(0, nil),
MinTime: 0,
MaxTime: int64(2 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(1, nil),
MinTime: int64(2 * time.Hour / time.Millisecond),
MaxTime: int64(4 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(2, nil),
MinTime: int64(4 * time.Hour / time.Millisecond),
MaxTime: int64(6 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(3, nil),
MinTime: int64(6 * time.Hour / time.Millisecond),
MaxTime: int64(8 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(4, nil),
MinTime: int64(8 * time.Hour / time.Millisecond),
MaxTime: int64(10 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(5, nil),
MinTime: int64(10 * time.Hour / time.Millisecond),
MaxTime: int64(12 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(6, nil),
MinTime: int64(12 * time.Hour / time.Millisecond),
MaxTime: int64(20 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(7, nil),
MinTime: int64(20 * time.Hour / time.Millisecond),
MaxTime: int64(28 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
createBlockMeta(0, 0, 2, map[string]string{"a": "1"}, 0),
createBlockMeta(1, 2, 4, map[string]string{"a": "1"}, 0),
createBlockMeta(2, 4, 6, map[string]string{"b": "2"}, 0),
createBlockMeta(3, 6, 8, map[string]string{"b": "2"}, 0),
createBlockMeta(4, 8, 10, map[string]string{"b": "2"}, 0),
createBlockMeta(5, 10, 12, map[string]string{"a": "1", "b": "2"}, 1),
createBlockMeta(6, 12, 20, map[string]string{"a": "1", "b": "2"}, 1),
createBlockMeta(7, 20, 28, map[string]string{"a": "1", "b": "2"}, 1),
},
expected: map[string]planResult{
keys[0]: {
Expand All @@ -329,75 +261,12 @@ func TestCompactProgressCalculate(t *testing.T) {
{
testName: "second_test",
input: []*metadata.Meta{
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(0, nil),
MinTime: 0,
MaxTime: int64(2 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(1, nil),
MinTime: int64(2 * time.Hour / time.Millisecond),
MaxTime: int64(4 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(2, nil),
MinTime: int64(4 * time.Hour / time.Millisecond),
MaxTime: int64(6 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(3, nil),
MinTime: int64(6 * time.Hour / time.Millisecond),
MaxTime: int64(10 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(4, nil),
MinTime: int64(10 * time.Hour / time.Millisecond),
MaxTime: int64(14 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(5, nil),
MinTime: int64(14 * time.Hour / time.Millisecond),
MaxTime: int64(16 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
createBlockMeta(0, 0, 2, map[string]string{"a": "1"}, 0),
createBlockMeta(1, 2, 4, map[string]string{"b": "2"}, 0),
createBlockMeta(2, 4, 6, map[string]string{"b": "2"}, 0),
createBlockMeta(3, 6, 10, map[string]string{"a": "1", "b": "2"}, 1),
createBlockMeta(4, 10, 14, map[string]string{"a": "1", "b": "2"}, 1),
createBlockMeta(5, 14, 16, map[string]string{"a": "1", "b": "2"}, 1),
},
expected: map[string]planResult{
keys[0]: {
Expand All @@ -417,63 +286,11 @@ func TestCompactProgressCalculate(t *testing.T) {
{
testName: "third_test",
input: []*metadata.Meta{
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(0, nil),
MinTime: 0,
MaxTime: int64(2 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(1, nil),
MinTime: int64(2 * time.Hour / time.Millisecond),
MaxTime: int64(4 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(2, nil),
MinTime: int64(4 * time.Hour / time.Millisecond),
MaxTime: int64(6 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"b": "2"},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(3, nil),
MinTime: int64(6 * time.Hour / time.Millisecond),
MaxTime: int64(8 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
{
BlockMeta: tsdb.BlockMeta{
ULID: ulid.MustNew(4, nil),
MinTime: int64(8 * time.Hour / time.Millisecond),
MaxTime: int64(10 * time.Hour / time.Millisecond),
},
Thanos: metadata.Thanos{
Version: 1,
Labels: map[string]string{"a": "1", "b": "2"},
Downsample: metadata.ThanosDownsample{Resolution: 1},
},
},
createBlockMeta(0, 0, 2, map[string]string{"a": "1"}, 0),
createBlockMeta(1, 2, 4, map[string]string{"b": "2"}, 0),
createBlockMeta(2, 4, 6, map[string]string{"b": "2"}, 0),
createBlockMeta(3, 6, 8, map[string]string{"a": "1", "b": "2"}, 1),
createBlockMeta(4, 8, 10, map[string]string{"a": "1", "b": "2"}, 1),
},
expected: map[string]planResult{
keys[0]: {
Expand All @@ -499,23 +316,17 @@ func TestCompactProgressCalculate(t *testing.T) {
}
// form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay
groups, err := grouper.Groups(blocks)
if err != nil {
level.Warn(logger).Log("msg, unable to form groups")
}
testutil.Ok(t, err)
ps := NewCompactionProgressCalculator(unRegisterer, planner)
err = ps.ProgressCalculate(context.Background(), groups)
testutil.Ok(t, err)
metrics := ps.CompactProgressMetrics
testutil.Ok(t, err)
for _, key := range keys {
for key := range tcase.expected {
a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues(key)
if err != nil {
level.Warn(logger).Log("msg", "could not get number of blocks")
}
testutil.Ok(t, err)
b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues(key)
if err != nil {
level.Warn(logger).Log("msg", "could not get number of runs")
}

testutil.Ok(t, err)
testutil.Equals(t, tcase.expected[key].compactionBlocks, promtestutil.ToFloat64(a))
testutil.Equals(t, tcase.expected[key].compactionRuns, promtestutil.ToFloat64(b))
}
Expand Down Expand Up @@ -778,9 +589,7 @@ func TestDownsampleProgressCalculate(t *testing.T) {
blocks[meta.ULID] = meta
}
groups, err := grouper.Groups(blocks)
if err != nil {
level.Warn(logger).Log("msg", "unable to form groups")
}
testutil.Ok(t, err)

ds := NewDownsampleProgressCalculator(unRegisterer)
err = ds.ProgressCalculate(context.Background(), groups)
Expand Down

0 comments on commit fe3355e

Please sign in to comment.