Merge pull request grafana#1202 from cortexproject/bigchunk-size-cap

Add a cap on bigChunk size
cyriltovena · Feb 5, 2019 · 9200617 · 9200617
2 parents b311ae7 + a350d41
commit 9200617
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 13 deletions.
diff --git a/encoding/bigchunk.go b/encoding/bigchunk.go
@@ -31,6 +31,9 @@ func newBigchunk() *bigchunk {
 
 func (b *bigchunk) Add(sample model.SamplePair) ([]Chunk, error) {
 	if b.remainingSamples == 0 {
+		if bigchunkSizeCapBytes > 0 && b.Size() > bigchunkSizeCapBytes {
+			return addToOverflowChunk(b, sample)
+		}
 		if err := b.addNextChunk(sample.Timestamp); err != nil {
 			return nil, err
 		}

diff --git a/encoding/factory.go b/encoding/factory.go
@@ -1,15 +1,30 @@
 package encoding
 
 import (
+	"flag"
 	"fmt"
 	"strconv"
 )
 
 // Encoding defines which encoding we are using, delta, doubledelta, or varbit
 type Encoding byte
 
-// DefaultEncoding can be changed via a flag.
-var DefaultEncoding = DoubleDelta
+// Config configures the behaviour of chunk encoding
+type Config struct{}
+
+var (
+	// DefaultEncoding exported for use in unit tests elsewhere
+	DefaultEncoding             = DoubleDelta
+	alwaysMarshalFullsizeChunks = true
+	bigchunkSizeCapBytes        = 0
+)
+
+// RegisterFlags registers configuration settings.
+func (Config) RegisterFlags(f *flag.FlagSet) {
+	f.Var(&DefaultEncoding, "ingester.chunk-encoding", "Encoding version to use for chunks.")
+	flag.BoolVar(&alwaysMarshalFullsizeChunks, "store.fullsize-chunks", alwaysMarshalFullsizeChunks, "When saving varbit chunks, pad to 1024 bytes")
+	flag.IntVar(&bigchunkSizeCapBytes, "store.bigchunk-size-cap-bytes", bigchunkSizeCapBytes, "When using bigchunk encoding, start a new bigchunk if over this size (0 = unlimited)")
+}
 
 // String implements flag.Value.
 func (e Encoding) String() string {

diff --git a/encoding/varbit.go b/encoding/varbit.go
@@ -18,7 +18,6 @@ package encoding
 
 import (
 	"encoding/binary"
-	"flag"
 	"fmt"
 	"io"
 	"math"
@@ -315,16 +314,6 @@ func (c varbitChunk) Utilization() float64 {
 	return math.Min(float64(c.nextSampleOffset()/8+15)/float64(cap(c)), 1)
 }
 
-// MarshalConfig configures the behaviour of marshalling
-type MarshalConfig struct{}
-
-var alwaysMarshalFullsizeChunks = true
-
-// RegisterFlags registers configuration settings.
-func (MarshalConfig) RegisterFlags(f *flag.FlagSet) {
-	flag.BoolVar(&alwaysMarshalFullsizeChunks, "store.fullsize-chunks", alwaysMarshalFullsizeChunks, "When saving varbit chunks, pad to 1024 bytes")
-}
-
 // marshalLen returns the number of bytes that should be marshalled for this chunk
 func (c varbitChunk) marshalLen() int {
 	bits := c.nextSampleOffset()