Fix poller to maintain previous blocklist results on tenant error

CHANGELOG.md

@@ -35,6 +35,8 @@
 * [BUGFIX] Fix http connection reuse on GCP and AWS by reading io.EOF through the http body. [#3760](https://github.com/grafana/tempo/pull/3760) (@bmteller)
 * [BUGFIX] Improved handling of complete blocks in localblocks processor after enabling flusing [#3805](https://github.com/grafana/tempo/pull/3805) (@mapno)
 * [BUGFIX] Handle out of boundaries spans kinds [#3861](https://github.com/grafana/tempo/pull/3861) (@javiermolinar)
+* [BUGFIX] Maintain previous tenant blocklist on tenant errors [#3860](https://github.com/grafana/tempo/pull/3860) (@zalegrala)
+
 
 ## v2.5.0
 

tempodb/blocklist/poller.go

@@ -162,6 +162,9 @@ func (p *Poller) Do(previous *List) (PerTenant, PerTenantCompacted, error) {
 				level.Error(p.logger).Log("msg", "exiting polling loop early because too many errors", "errCount", consecutiveErrors)
 				return nil, nil, err
 			}
+
+			blocklist[tenantID] = previous.Metas(tenantID)
+			compactedBlocklist[tenantID] = previous.CompactedMetas(tenantID)
 			continue
 		}
 

tempodb/blocklist/poller_test.go

@@ -609,6 +609,8 @@ func TestPollComparePreviousResults(t *testing.T) {
 		expectedBlockMetaCalls          map[string]map[uuid.UUID]int
 		expectedCompactedBlockMetaCalls map[string]map[uuid.UUID]int
 
+		tollerateErrors int
+
 		readerErr bool
 		err       error
 	}{
@@ -761,6 +763,39 @@ func TestPollComparePreviousResults(t *testing.T) {
 			expectedCompactedPerTenant: PerTenantCompacted{},
 			expectedBlockMetaCalls:     map[string]map[uuid.UUID]int{},
 		},
+		{
+			name:            "previous results with read error should maintain previous results",
+			tollerateErrors: 10,
+			readerErr:       true,
+			// err:               fmt.Errorf("failed to poll tenant blocks"),
+			previousPerTenant: PerTenant{},
+			previousCompactedPerTenant: PerTenantCompacted{
+				"test": []*backend.CompactedBlockMeta{
+					{BlockMeta: backend.BlockMeta{BlockID: zero}},
+					{BlockMeta: backend.BlockMeta{BlockID: aaa}},
+					{BlockMeta: backend.BlockMeta{BlockID: eff}},
+				},
+			},
+			currentPerTenant: PerTenant{},
+			currentCompactedPerTenant: PerTenantCompacted{
+				"test": []*backend.CompactedBlockMeta{
+					{BlockMeta: backend.BlockMeta{BlockID: zero}},
+					{BlockMeta: backend.BlockMeta{BlockID: aaa}},
+					{BlockMeta: backend.BlockMeta{BlockID: eff}},
+				},
+			},
+			expectedPerTenant: PerTenant{
+				"test": []*backend.BlockMeta{},
+			},
+			expectedCompactedPerTenant: PerTenantCompacted{
+				"test": []*backend.CompactedBlockMeta{
+					{BlockMeta: backend.BlockMeta{BlockID: zero}},
+					{BlockMeta: backend.BlockMeta{BlockID: aaa}},
+					{BlockMeta: backend.BlockMeta{BlockID: eff}},
+				},
+			},
+			expectedBlockMetaCalls: map[string]map[uuid.UUID]int{},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -775,9 +810,10 @@ func TestPollComparePreviousResults(t *testing.T) {
 
 			// This mock reader returns error or nil based on the tenant ID
 			poller := NewPoller(&PollerConfig{
-				PollConcurrency:     testPollConcurrency,
-				PollFallback:        testPollFallback,
-				TenantIndexBuilders: testBuilders,
+				PollConcurrency:           testPollConcurrency,
+				PollFallback:              testPollFallback,
+				TenantIndexBuilders:       testBuilders,
+				TolerateConsecutiveErrors: tc.tollerateErrors,
 			}, s, r, c, w, log.NewNopLogger())
 
 			metas, compactedMetas, err := poller.Do(previous)