roachtest: adding backup/restore tests for minio

pkg/cmd/roachtest/tests/BUILD.bazel

@@ -168,6 +168,7 @@ go_library(
         "rust_postgres_blocklist.go",
         "s3_clone_backup_restore.go",
         "s3_microceph.go",
+        "s3_minio.go",
         "schemachange.go",
         "schemachange_random_load.go",
         "scrub.go",

pkg/cmd/roachtest/tests/s3_clone_backup_restore.go

@@ -10,6 +10,8 @@ import (
 	gosql "database/sql"
 	"fmt"
 	"math/rand"
+	"os"
+	"path"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
@@ -65,6 +67,42 @@ func registerBackupS3Clones(r registry.Registry) {
 			},
 		})
 	}
+
+	r.Add(registry.TestSpec{
+		Name:                      "backup/minio",
+		Owner:                     registry.OwnerFieldEng,
+		Cluster:                   r.MakeClusterSpec(4, spec.WorkloadNodeCount(1)),
+		EncryptionSupport:         registry.EncryptionMetamorphic,
+		Leases:                    registry.MetamorphicLeases,
+		CompatibleClouds:          registry.Clouds(spec.GCE),
+		Suites:                    registry.Suites(registry.Nightly),
+		TestSelectionOptOutSuites: registry.Suites(registry.Nightly),
+		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
+			v := s3BackupRestoreValidator{
+				t:            t,
+				c:            c,
+				crdbNodes:    c.CRDBNodes(),
+				csvPort:      8081,
+				importNode:   c.Node(1),
+				rows:         1000,
+				workloadNode: c.WorkloadNode(),
+			}
+			v.startCluster(ctx)
+			mgr := minioManager{
+				t:      t,
+				c:      c,
+				bucket: backupTestingBucket,
+				// For now, we use the workload node as the minio cluster
+				minioNodes: c.Node(c.Spec().NodeCount),
+				key:        randomString(32),
+				secret:     randomString(64),
+			}
+			mgr.install(ctx)
+			defer mgr.cleanup(ctx)
+			v.validateBackupRestore(ctx, mgr)
+		},
+	})
+
 }
 
 // s3Provider defines the methods that the S3 object store has to provide
@@ -229,6 +267,32 @@ func (v *s3BackupRestoreValidator) runWorload(ctx context.Context, duration time
 	return v.c.RunE(ctx, option.WithNodes(v.workloadNode), cmd)
 }
 
+func installCa(ctx context.Context, t test.Test, c cluster.Cluster) error {
+	localCertsDir, err := os.MkdirTemp("", "roachtest-certs")
+	if err != nil {
+		return err
+	}
+	// get the ca file from one of the nodes.
+	caFile := path.Join(localCertsDir, "ca.crt")
+	conn := c.Conn(ctx, t.L(), 1)
+	defer conn.Close()
+	if err := c.Get(ctx, t.L(), "certs/ca.crt", caFile, c.Node(1)); err != nil {
+		return err
+	}
+	caCert, err := os.ReadFile(caFile)
+	if err != nil {
+		return err
+	}
+	// Disabling caching for Custom CA, see https://github.com/cockroachdb/cockroach/issues/125051.
+	if _, err := conn.ExecContext(ctx, "set cluster setting cloudstorage.s3.session_reuse.enabled = false"); err != nil {
+		return err
+	}
+	if _, err := conn.ExecContext(ctx, "set cluster setting cloudstorage.http.custom_ca=$1", caCert); err != nil {
+		return err
+	}
+	return nil
+}
+
 // randomString returns a random string with the given size.
 func randomString(size int) string {
 	var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

pkg/cmd/roachtest/tests/s3_microceph.go

@@ -9,8 +9,6 @@ import (
 	"context"
 	"fmt"
 	"net/url"
-	"os"
-	"path"
 	"path/filepath"
 
 	"github.com/cockroachdb/cockroach/pkg/cloud/amazon"
@@ -38,7 +36,7 @@ for l in  a b c; do
   sudo microceph disk add --wipe "/dev/sdi${l}"
 done`
 
-// cephCleanup remove microceph and the loop devices.
+// cephCleanup removes microceph and the loop devices.
 const cephCleanup = `
 #!/bin/bash
 sudo microceph disable rgw
@@ -147,29 +145,7 @@ func (m cephManager) maybeInstallCa(ctx context.Context) error {
 	if !m.secure {
 		return nil
 	}
-	localCertsDir, err := os.MkdirTemp("", "roachtest-certs")
-	if err != nil {
-		return err
-	}
-	// get the ca file from one of the nodes.
-	caFile := path.Join(localCertsDir, "ca.crt")
-	conn := m.c.Conn(ctx, m.t.L(), 1)
-	defer conn.Close()
-	if err := m.c.Get(ctx, m.t.L(), "certs/ca.crt", caFile, m.c.Node(1)); err != nil {
-		return err
-	}
-	caCert, err := os.ReadFile(caFile)
-	if err != nil {
-		return err
-	}
-	// Disabling caching for Custom CA, see https://github.com/cockroachdb/cockroach/issues/125051.
-	if _, err := conn.ExecContext(ctx, "set cluster setting cloudstorage.s3.session_reuse.enabled = false"); err != nil {
-		return err
-	}
-	if _, err := conn.ExecContext(ctx, "set cluster setting cloudstorage.http.custom_ca=$1", caCert); err != nil {
-		return err
-	}
-	return nil
+	return installCa(ctx, m.t, m.c)
 }
 
 // put creates a file in the ceph node with the given content.

pkg/cmd/roachtest/tests/s3_minio.go

@@ -0,0 +1,93 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package tests
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"path"
+
+	"github.com/cockroachdb/cockroach/pkg/cloud/amazon"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+)
+
+// minioDir is the directory for supporting files.
+var minioDir = "/tmp/minio"
+
+// minioManager manages a single node minio cluster, used to
+// validate the backup and restore functionality.
+type minioManager struct {
+	t          test.Test
+	c          cluster.Cluster
+	bucket     string
+	minioNodes option.NodeListOption // The nodes within the cluster used by Minio.
+	key        string
+	secret     string
+}
+
+// minioManager implements s3Provider
+var _ s3Provider = &minioManager{}
+
+// getBackupURI implements s3Provider.
+func (m minioManager) getBackupURI(ctx context.Context, dest string) (string, error) {
+	addr, err := m.c.InternalIP(ctx, m.t.L(), m.minioNodes)
+	if err != nil {
+		return "", err
+	}
+	m.t.Status("minio: ", addr)
+	endpointURL := `https://` + addr[0]
+
+	q := make(url.Values)
+	q.Add(amazon.AWSAccessKeyParam, m.key)
+	q.Add(amazon.AWSSecretParam, m.secret)
+	q.Add(amazon.AWSUsePathStyle, "true")
+	// Region is required in the URL, but not used in Minio.
+	q.Add(amazon.S3RegionParam, "dummy")
+	q.Add(amazon.AWSEndpointParam, endpointURL)
+	uri := fmt.Sprintf("s3://%s/%s?%s", m.bucket, dest, q.Encode())
+	return uri, nil
+}
+
+func (m minioManager) cleanup(ctx context.Context) {
+	m.run(ctx, "removing minio", "sudo docker rm -f minio")
+	m.run(ctx, "removing minio dir", fmt.Sprintf(`rm  -rf %s`, minioDir))
+}
+
+// install a single node minio cluster within a docker container.
+// It is fatal on errors.
+func (m minioManager) install(ctx context.Context) {
+	if err := m.c.Install(ctx, m.t.L(), m.minioNodes, "docker"); err != nil {
+		m.t.Fatalf("failed to install docker: %v", err)
+	}
+	certsDir := path.Join(minioDir, "certs")
+	m.run(ctx, `copy CA`,
+		fmt.Sprintf(`mkdir -p %[1]s/CAs ; cp certs/ca.crt %[1]s/CAs/ca.crt; `, certsDir))
+	m.run(ctx, `copy certs/key`,
+		fmt.Sprintf(`cp certs/node.crt %[1]s/public.crt; cp certs/node.key  %[1]s/private.key; `,
+			certsDir))
+	m.run(ctx, `installing minio`,
+		fmt.Sprintf(`sudo docker run --name minio -d -p 443:9000 -e "MINIO_ROOT_USER=%s" -e "MINIO_ROOT_PASSWORD=%s" --privileged -v %s:/root/.minio minio/minio server  /data`,
+			m.key, m.secret, minioDir))
+
+	m.run(ctx, `install s3cmd`, `sudo apt install -y s3cmd`)
+	m.run(ctx, `creating bucket`,
+		fmt.Sprintf(s3cmdSsl, m.key, m.secret, "mb s3://"+m.bucket))
+
+	if err := installCa(ctx, m.t, m.c); err != nil {
+		m.t.Fatal(err)
+	}
+}
+
+// run the given command on the minio node.
+func (m minioManager) run(ctx context.Context, msg string, cmd ...string) {
+	m.t.Status(msg, "...")
+	m.t.Status(cmd)
+	m.c.Run(ctx, option.WithNodes(m.minioNodes), cmd...)
+	m.t.Status(msg, " done")
+}