Pseudo randomize push check tag

When multiple Flux instances are synchronizing with the same repository
but different branches, and the upstream has some latency, they get
confused about the write check they are performing due to the tag
being overwritten with a commit hash that does not belong to the branch
they are synchronzing with.

This commit pseudo randomizes the tag used to check if we can write to
the upstream repository and removes the `--force` options from the tag
and push commands so that the tag is never accidentally overwritten and
multiple instances no longer get in each other's way.

pkg/git/operations.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -98,19 +99,26 @@ func add(ctx context.Context, workingDir, path string) error {
 // (being able to `clone` is an adequate check that we can read the
 // upstream).
 func checkPush(ctx context.Context, workingDir, upstream, branch string) error {
-	// --force just in case we fetched the tag from upstream when cloning
-	args := []string{"tag", "--force", CheckPushTag}
+	// we need to pseudo randomize the tag we use for the write check
+	// as multiple Flux instances can perform the check simultaneously
+	// for different branches, causing commit reference conflicts
+	b := make([]byte, 5)
+	if _, err := rand.Read(b); err != nil {
+		return err
+	}
+	pseudoRandPushTag := fmt.Sprintf("%s-%x", CheckPushTagPrefix, b)
+	args := []string{"tag", pseudoRandPushTag}
 	if branch != "" {
 		args = append(args, branch)
 	}
 	if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
 		return errors.Wrap(err, "tag for write check")
 	}
-	args = []string{"push", "--force", upstream, "tag", CheckPushTag}
+	args = []string{"push", upstream, "tag", pseudoRandPushTag}
 	if err := execGitCmd(ctx, args, gitCmdConfig{dir: workingDir}); err != nil {
 		return errors.Wrap(err, "attempt to push tag")
 	}
-	return deleteTag(ctx, workingDir, CheckPushTag, upstream)
+	return deleteTag(ctx, workingDir, pseudoRandPushTag, upstream)
 }
 
 // deleteTag deletes the given git tag

pkg/git/repo.go

@@ -15,7 +15,7 @@ const (
 	defaultInterval = 5 * time.Minute
 	defaultTimeout  = 20 * time.Second
 
-	CheckPushTag = "flux-write-check"
+	CheckPushTagPrefix = "flux-write-check"
 )
 
 var (