Clear completed, partial blocks on generator startup (#3704)

* Clear completed, partial blocks on generator startup * Update changelog
grafana · May 23, 2024 · 7c2e7c7 · 7c2e7c7
1 parent 04b7a5a
commit 7c2e7c7
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -70,6 +70,7 @@
 * [BUGFIX] Correctly parse traceql queries with > 1024 character attribute names or static values. [#3571](https://github.com/grafana/tempo/issues/3571) (@joe-elliott)
 * [BUGFIX] Fix span-metrics' subprocessors bug that applied wrong configs when running multiple tenants. [#3612](https://github.com/grafana/tempo/pull/3612) (@mapno)
 * [BUGFIX] Fix panic in query-frontend when combining results [#3683](https://github.com/grafana/tempo/pull/3683) (@mapno)
+* [BUGFIX] Fix panic in metrics-generator when starting with a partial completed block [#3704](https://github.com/grafana/tempo/pull/3704) (@zalegrala)
 
 ## v2.4.2
 

diff --git a/modules/generator/processor/localblocks/processor.go b/modules/generator/processor/localblocks/processor.go
@@ -3,6 +3,7 @@ package localblocks
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"sort"
@@ -720,7 +721,15 @@ func (p *Processor) reloadBlocks() error {
 	for _, id := range ids {
 		meta, err := r.BlockMeta(ctx, id, t)
 
-		if errors.Is(err, backend.ErrDoesNotExist) {
+		var clearBlock bool
+		if err != nil {
+			var vv *json.SyntaxError
+			if errors.Is(err, backend.ErrDoesNotExist) || errors.As(err, &vv) {
+				clearBlock = true
+			}
+		}
+
+		if clearBlock {
 			// Partially written block, delete and continue
 			err = l.ClearBlock(id, t)
 			if err != nil {

diff --git a/modules/generator/processor/localblocks/processor_test.go b/modules/generator/processor/localblocks/processor_test.go
@@ -2,16 +2,20 @@ package localblocks
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/grafana/tempo/pkg/tempopb"
 	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
 	"github.com/grafana/tempo/pkg/util/test"
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/encoding"
 	"github.com/grafana/tempo/tempodb/encoding/common"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
 	"github.com/grafana/tempo/tempodb/wal"
 	"github.com/stretchr/testify/require"
 )
@@ -198,6 +202,75 @@ func TestReplicationFactor(t *testing.T) {
 	}
 }
 
+func TestBadBlocks(t *testing.T) {
+	wal, err := wal.New(&wal.Config{
+		Filepath: t.TempDir(),
+		Version:  encoding.DefaultEncoding().Version(),
+	})
+	require.NoError(t, err)
+
+	u1 := uuid.New().String()
+
+	// Write some bad wal data
+	writeBadJSON(t,
+		filepath.Join(
+			wal.GetFilepath(),
+			u1+"+test-tenant+"+vparquet3.VersionString,
+			backend.MetaName,
+		),
+	)
+
+	writeBadJSON(t,
+		filepath.Join(
+			wal.GetFilepath(),
+			u1+"+test-tenant+"+vparquet3.VersionString,
+			"0000000001",
+		),
+	)
+
+	writeBadJSON(t,
+		filepath.Join(
+			wal.GetFilepath(),
+			uuid.New().String()+"+test-tenant+"+vparquet3.VersionString,
+			"0000000001",
+		),
+	)
+
+	// write a bad block meta for a completed block
+	writeBadJSON(t,
+		filepath.Join(
+			wal.GetFilepath(),
+			"blocks",
+			"test-tenant",
+			uuid.New().String(),
+			backend.MetaName,
+		),
+	)
+
+	cfg := Config{
+		FlushCheckPeriod:     time.Minute,
+		TraceIdlePeriod:      time.Minute,
+		CompleteBlockTimeout: time.Minute,
+		Block: &common.BlockConfig{
+			Version: encoding.DefaultEncoding().Version(),
+		},
+	}
+
+	_, err = New(cfg, "test-tenant", wal, &mockOverrides{})
+	require.NoError(t, err)
+}
+
 func verifyReplicationFactor(t *testing.T, b common.BackendBlock) {
 	require.Equal(t, 1, int(b.BlockMeta().ReplicationFactor))
 }
+
+func writeBadJSON(t *testing.T, path string) {
+	dir := filepath.Dir(path)
+	err := os.MkdirAll(dir, 0o700)
+	require.NoError(t, err)
+	f, err := os.Create(path)
+	require.NoError(t, err)
+	defer f.Close()
+	_, err = f.WriteString("{")
+	require.NoError(t, err)
+}