cmd/thanos/receive: avoid deadlock (#1727)

While debugging #1721, I found that when thanos receive bails, there is a race in a select statement, where the non-returning branch may be chosen. This branch will deadlock if selected twice because the channel reader has already exited. The way to prevent this is by checking if we need to exit on every loop. Signed-off-by: Lucas Servén Marín <[email protected]>
thanos-io · Nov 7, 2019 · 514f308 · 514f308
1 parent 27a578d
commit 514f308
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go
@@ -248,7 +248,6 @@ func runReceive(
 			defer func() {
 				if err := db.Flush(); err != nil {
 					level.Warn(logger).Log("err", err, "msg", "failed to flush storage")
-					return
 				}
 				if err := db.Close(); err != nil {
 					level.Warn(logger).Log("err", err, "msg", "failed to close storage")
@@ -449,6 +448,11 @@ func runReceive(
 				}()
 				defer close(uploadDone)
 				for {
+					select {
+					case <-ctx.Done():
+						return nil
+					default:
+					}
 					select {
 					case <-ctx.Done():
 						return nil