diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go index ed8a7efba2..a89e24c2c5 100644 --- a/pkg/receive/multitsdb_test.go +++ b/pkg/receive/multitsdb_test.go @@ -166,11 +166,11 @@ func TestMultiTSDB(t *testing.T) { var ( expectedFooResp = &storepb.Series{ Labels: []labelpb.ZLabel{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "replica", Value: "01"}, {Name: "tenant_id", Value: "foo"}}, - Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@\003L\235\2354X\315\001\330\r\257Mui\251\327:U")}}}, + Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@\003L\235\2354X\315\001\330\r\257Mui\251\327:U"), Hash: 9768694233508509040}}}, } expectedBarResp = &storepb.Series{ Labels: []labelpb.ZLabel{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "replica", Value: "01"}, {Name: "tenant_id", Value: "bar"}}, - Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@4i\223\263\246\213\032\001\330\035i\337\322\352\323S\256t\270")}}}, + Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@4i\223\263\246\213\032\001\330\035i\337\322\352\323S\256t\270"), Hash: 2304287992246504442}}}, } ) diff --git a/pkg/store/proxy_heap.go b/pkg/store/proxy_heap.go index 822633b6a7..558ffcb69b 100644 --- a/pkg/store/proxy_heap.go +++ b/pkg/store/proxy_heap.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" + "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/tracing" @@ -130,15 +131,16 @@ func (d *dedupResponseHeap) At() *storepb.SeriesResponse { if field == nil { continue } - h := xxhash.Sum64(field.Data) + hash := field.Hash + if hash == 0 { + hash = xxhash.Sum64(field.Data) + } - if _, ok := chunkDedupMap[h]; !ok { + if _, ok := chunkDedupMap[hash]; !ok { chk := chk - - chunkDedupMap[h] = &chk + chunkDedupMap[hash] = &chk } } - } } diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go index ad0939687f..8a997907e4 100644 --- a/pkg/store/proxy_test.go +++ b/pkg/store/proxy_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/cespare/xxhash/v2" "github.com/go-kit/log" "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" @@ -2079,6 +2080,7 @@ func TestDedupRespHeap_Deduplication(t *testing.T) { { Raw: &storepb.Chunk{ Type: storepb.Chunk_XOR, + Hash: xxhash.Sum64([]byte(`abcdefgh`)), Data: []byte(`abcdefgh`), }, },