Use pre-calculated hashes (#5817)

Thanos stores can now calculate chunk hashes and send them as part of the series response. This commit modifies the dedup proxy to not calculate those hashes if they are available. Signed-off-by: Filip Petkovski <[email protected]> Signed-off-by: Filip Petkovski <[email protected]>
thanos-io · Oct 25, 2022 · 54f6d4a · 54f6d4a
1 parent 2d0407f
commit 54f6d4a
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 7 deletions.
diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go
@@ -166,11 +166,11 @@ func TestMultiTSDB(t *testing.T) {
 var (
 	expectedFooResp = &storepb.Series{
 		Labels: []labelpb.ZLabel{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "replica", Value: "01"}, {Name: "tenant_id", Value: "foo"}},
-		Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@\003L\235\2354X\315\001\330\r\257Mui\251\327:U")}}},
+		Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@\003L\235\2354X\315\001\330\r\257Mui\251\327:U"), Hash: 9768694233508509040}}},
 	}
 	expectedBarResp = &storepb.Series{
 		Labels: []labelpb.ZLabel{{Name: "a", Value: "1"}, {Name: "b", Value: "2"}, {Name: "replica", Value: "01"}, {Name: "tenant_id", Value: "bar"}},
-		Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@4i\223\263\246\213\032\001\330\035i\337\322\352\323S\256t\270")}}},
+		Chunks: []storepb.AggrChunk{{MinTime: 1, MaxTime: 3, Raw: &storepb.Chunk{Data: []byte("\000\003\002@4i\223\263\246\213\032\001\330\035i\337\322\352\323S\256t\270"), Hash: 2304287992246504442}}},
 	}
 )
 

diff --git a/pkg/store/proxy_heap.go b/pkg/store/proxy_heap.go
@@ -21,6 +21,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/model/labels"
+
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/tracing"
@@ -130,15 +131,16 @@ func (d *dedupResponseHeap) At() *storepb.SeriesResponse {
 				if field == nil {
 					continue
 				}
-				h := xxhash.Sum64(field.Data)
+				hash := field.Hash
+				if hash == 0 {
+					hash = xxhash.Sum64(field.Data)
+				}
 
-				if _, ok := chunkDedupMap[h]; !ok {
+				if _, ok := chunkDedupMap[hash]; !ok {
 					chk := chk
-
-					chunkDedupMap[h] = &chk
+					chunkDedupMap[hash] = &chk
 				}
 			}
-
 		}
 	}
 

diff --git a/pkg/store/proxy_test.go b/pkg/store/proxy_test.go
@@ -15,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/go-kit/log"
 	"github.com/gogo/protobuf/proto"
 	"github.com/gogo/protobuf/types"
@@ -2079,6 +2080,7 @@ func TestDedupRespHeap_Deduplication(t *testing.T) {
 								{
 									Raw: &storepb.Chunk{
 										Type: storepb.Chunk_XOR,
+										Hash: xxhash.Sum64([]byte(`abcdefgh`)),
 										Data: []byte(`abcdefgh`),
 									},
 								},