fix: influx gzip writes #2479
fix: influx gzip writes #2479
Conversation
| @@ -571,6 +568,33 @@ func (p Point) Bytes(precision string) []byte { | |||
| return bytes | |||
| } | |||
|
|
|||
| func (p Point) BytesWithLineFeed(precision string) []byte { | |||
| key := imodels.MakeKey([]byte(p.Name), imodels.NewTags(p.Tags)) | |||
There was a problem hiding this comment.
This is a copy of the func (p Point) Bytes method, only it adds a line feed to the end of the line to avoid the allocation of doing it afterwards.
| @@ -196,6 +197,59 @@ func TestClient_Write(t *testing.T) { | |||
| } | |||
| } | |||
|
|
|||
| func TestClient_WriteLarge(t *testing.T) { | |||
| expected := &bytes.Buffer{} | |||
There was a problem hiding this comment.
This writes a million points to an httptest server and checks to make sure that it is delivered properly. The million points are gzipped.
| @@ -78,5 +78,10 @@ func TestPoint_Bytes(t *testing.T) { | |||
| t.Errorf("%s: Bytes() mismatch:\n actual: %v\n exp: %v", | |||
| test.name, got, exp) | |||
| } | |||
| if got, exp := string(p.BytesWithLineFeed(test.precision)), test.exp+"\n"; got != exp { | |||
| t.Errorf("%s: Bytes() mismatch:\n actual: %v\n exp: %v", | |||
There was a problem hiding this comment.
We want to test not just the Point.Bytes() but also Point.BytesWithLineFeed()
| return err | ||
| } | ||
|
|
||
| if err := b.WriteByte('\n'); err != nil { |
There was a problem hiding this comment.
It's very surprising to me that this would be allocating - what is the allocation caused by?
There was a problem hiding this comment.
That line doesn't allocate, but the gzip writer doesn't have a WriteByte, it only has Write, which means allocating a single element byte slice.
This is a fix for a data race in kapacitor 1.5.8 where large batch writes were not completely writing to influxdb.