Add retry functionality

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/cockroachdb/apd v1.1.0 // indirect
 	github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c
 	github.com/coreos/go-semver v0.2.0
-	github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect
 	github.com/denisenkom/go-mssqldb v0.0.0-20190412130859-3b1d194e553a
 	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
@@ -49,7 +48,6 @@ require (
 	github.com/golang/protobuf v1.3.2
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-metrics-stackdriver v0.0.0-20190816035513-b52628e82e2a
-	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
 	github.com/hashicorp/consul-template v0.22.0
 	github.com/hashicorp/consul/api v1.1.0

plugins/database/mongodb/mongodb.go

@@ -132,9 +132,8 @@ func (m *MongoDB) CreateUser(ctx context.Context, statements dbplugin.Statements
 		Roles:    mongoCS.Roles.toStandardRolesArray(),
 	}
 
-	result := client.Database(mongoCS.DB).RunCommand(ctx, createUserCmd, nil)
-	if result.Err() != nil {
-		return "", "", result.Err()
+	if err := runCommandWithRetry(ctx, client, mongoCS.DB, createUserCmd); err != nil {
+		return "", "", err
 	}
 
 	return username, password, nil
@@ -168,9 +167,8 @@ func (m *MongoDB) SetCredentials(ctx context.Context, statements dbplugin.Statem
 	if err != nil {
 		return "", "", err
 	}
-	result := client.Database(cs.Database).RunCommand(ctx, changeUserCmd, nil)
-	if result.Err() != nil {
-		return "", "", result.Err()
+	if err := runCommandWithRetry(ctx, client, cs.Database, changeUserCmd); err != nil {
+		return "", "", err
 	}
 
 	return username, password, nil
@@ -224,11 +222,28 @@ func (m *MongoDB) RevokeUser(ctx context.Context, statements dbplugin.Statements
 		WriteConcern: writeconcern.New(writeconcern.WMajority()),
 	}
 
-	result := client.Database(db).RunCommand(ctx, dropUserCmd, nil)
-	return result.Err()
+	return runCommandWithRetry(ctx, client, db, dropUserCmd)
 }
 
 // RotateRootCredentials is not currently supported on MongoDB
 func (m *MongoDB) RotateRootCredentials(ctx context.Context, statements []string) (map[string]interface{}, error) {
 	return nil, errors.New("root credential rotation is not currently implemented in this database secrets engine")
 }
+
+// runCommandWithRetry runs a command with retry.
+func runCommandWithRetry(ctx context.Context, client *mongo.Client, db string, cmd interface{}) error {
+	timeout := time.Now().Add(1 * time.Minute)
+	for {
+		// Run command
+		result := client.Database(db).RunCommand(ctx, cmd, nil)
+		if result.Err() == nil {
+			break
+		}
+
+		if time.Now().After(timeout) {
+			return result.Err()
+		}
+		time.Sleep(30 * time.Millisecond)
+	}
+	return nil
+}