Address review issues

Signed-off-by: Kemal Akkoyun <[email protected]>

cmd/thanos/receive.go

@@ -87,7 +87,7 @@ func registerReceive(m map[string]setupFunc, app *kingpin.Application) {
 	tsdbMinBlockDuration := modelDuration(cmd.Flag("tsdb.min-block-duration", "Min duration for local TSDB blocks").Default("2h").Hidden())
 	tsdbMaxBlockDuration := modelDuration(cmd.Flag("tsdb.max-block-duration", "Max duration for local TSDB blocks").Default("2h").Hidden())
 	walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool()
-	noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory.").Default("false").Bool()
+	noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup.").Default("false").Bool()
 
 	ignoreBlockSize := cmd.Flag("shipper.ignore-unequal-block-size", "If true receive will not require min and max block size flags to be set to the same value. Only use this if you want to keep long retention and compaction enabled, as in the worst case it can result in ~2h data loss for your Thanos bucket storage.").Default("false").Hidden().Bool()
 	allowOutOfOrderUpload := cmd.Flag("shipper.allow-out-of-order-uploads",
@@ -304,9 +304,9 @@ func runReceive(
 			Help: "Number of Multi DB completed reloads with flush and potential upload due to hashring changes",
 		})
 
-		level.Debug(logger).Log("msg", "cleaning storage lock files")
-		if err := dbs.Clean(); err != nil {
-			return errors.Wrap(err, "cleaning storage lock files")
+		level.Debug(logger).Log("msg", "removing storage lock files if any")
+		if err := dbs.RemoveLockFilesIfAny(); err != nil {
+			return errors.Wrap(err, "remove storage lock files")
 		}
 
 		// TSDBs reload logic, listening on hashring changes.

cmd/thanos/rule.go

@@ -8,6 +8,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -78,7 +79,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application) {
 		Default("2h"))
 	tsdbRetention := modelDuration(cmd.Flag("tsdb.retention", "Block retention time on local disk.").
 		Default("48h"))
-	noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory.").Default("false").Bool()
+	noLockFile := cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup.").Default("false").Bool()
 	walCompression := cmd.Flag("tsdb.wal-compression", "Compress the tsdb WAL.").Default("true").Bool()
 
 	alertmgrs := cmd.Flag("alertmanagers.url", "Alertmanager replica URLs to push firing alerts. Ruler claims success if push to at least one alertmanager from discovered succeeds. The scheme should not be empty e.g `http` might be used. The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Alertmanager IPs through respective DNS lookups. The port defaults to 9093 or the SRV record's value. The URL path is used as a prefix for the regular Alertmanager API path.").
@@ -351,6 +352,12 @@ func runRule(
 	if err != nil {
 		return errors.Wrap(err, "open TSDB")
 	}
+
+	level.Debug(logger).Log("msg", "removing storage lock file if any")
+	if err := removeLockfileIfAny(logger, dataDir); err != nil {
+		return errors.Wrap(err, "remove storage lock files")
+	}
+
 	{
 		done := make(chan struct{})
 		g.Add(func() error {
@@ -643,6 +650,21 @@ func runRule(
 	return nil
 }
 
+func removeLockfileIfAny(logger log.Logger, dataDir string) error {
+	absdir, err := filepath.Abs(dataDir)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(filepath.Join(absdir, "lock")); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	level.Info(logger).Log("msg", "a leftover lockfile found and removed")
+	return nil
+}
+
 func parseFlagLabels(s []string) (labels.Labels, error) {
 	var lset labels.Labels
 	for _, l := range s {

docs/components/receive.md

@@ -160,5 +160,7 @@ Flags:
                                  requests.
       --tsdb.wal-compression     Compress the tsdb WAL.
       --tsdb.no-lockfile         Do not create lockfile in TSDB data directory.
+                                 In any case, the lockfiles will be deleted on
+                                 next startup.
 
 ```

docs/components/rule.md

@@ -283,6 +283,8 @@ Flags:
       --tsdb.block-duration=2h   Block duration for TSDB block.
       --tsdb.retention=48h       Block retention time on local disk.
       --tsdb.no-lockfile         Do not create lockfile in TSDB data directory.
+                                 In any case, the lockfiles will be deleted on
+                                 next startup.
       --tsdb.wal-compression     Compress the tsdb WAL.
       --alertmanagers.url=ALERTMANAGERS.URL ...
                                  Alertmanager replica URLs to push firing

pkg/receive/multitsdb.go

@@ -214,7 +214,7 @@ func (t *MultiTSDB) Sync(ctx context.Context) error {
 	return merr.Err()
 }
 
-func (t *MultiTSDB) Clean() error {
+func (t *MultiTSDB) RemoveLockFilesIfAny() error {
 	fis, err := ioutil.ReadDir(t.dataDir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -228,12 +228,14 @@ func (t *MultiTSDB) Clean() error {
 		if !fi.IsDir() {
 			continue
 		}
-		if err = os.Remove(filepath.Join(t.defaultTenantDataDir(fi.Name()), "lock")); err != nil {
+		if err := os.Remove(filepath.Join(t.defaultTenantDataDir(fi.Name()), "lock")); err != nil {
 			if os.IsNotExist(err) {
 				continue
 			}
 			merr.Add(err)
+			continue
 		}
+		level.Info(t.logger).Log("msg", "a leftover lockfile found and removed", "tenant", fi.Name())
 	}
 	return merr.Err()
 }