start minio server with tls (#4991)

* start minio server with tls Signed-off-by: clyang82 <[email protected]> * update TLSConfig package Signed-off-by: clyang82 <[email protected]> * return error if NewMinio has errors Signed-off-by: clyang82 <[email protected]> * fix lint issue Signed-off-by: clyang82 <[email protected]> * Fix weak cryptographic key in code scanning Signed-off-by: clyang82 <[email protected]> * workaround for flakiness of readiness Signed-off-by: clyang82 <[email protected]> * fix rebase issue Signed-off-by: clyang82 <[email protected]> * update query_test with new e2ethanos.NewMinio func Signed-off-by: clyang82 <[email protected]> * sleep 1 to see if minio can be ready or not Signed-off-by: clyang82 <[email protected]>
thanos-io · Jan 15, 2022 · 4d5f4d9 · 4d5f4d9
1 parent b97c19f
commit 4d5f4d9
Show file tree

Hide file tree

Showing 7 changed files with 175 additions and 171 deletions.
diff --git a/pkg/objstore/s3/s3_e2e_test.go b/pkg/objstore/s3/s3_e2e_test.go
@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/efficientgo/e2e"
-	e2edb "github.com/efficientgo/e2e/db"
 	"github.com/go-kit/log"
 
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
@@ -29,16 +28,12 @@ func BenchmarkUpload(b *testing.B) {
 	b.Cleanup(e2ethanos.CleanScenario(b, e))
 
 	const bucket = "benchmark"
-	m := e2ethanos.NewMinio(e, "benchmark", bucket)
+	m, err := e2ethanos.NewMinio(e, "benchmark", bucket)
+	testutil.Ok(b, err)
 	testutil.Ok(b, e2e.StartAndWaitReady(m))
 
-	bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(), s3.Config{
-		Bucket:    bucket,
-		AccessKey: e2edb.MinioAccessKey,
-		SecretKey: e2edb.MinioSecretKey,
-		Endpoint:  m.Endpoint("http"),
-		Insecure:  true,
-	}, "test-feed")
+	bkt, err := s3.NewBucketWithConfig(log.NewNopLogger(),
+		e2ethanos.NewS3Config(bucket, m.Endpoint("https"), e.SharedDir()), "test-feed")
 	testutil.Ok(b, err)
 
 	buf := bytes.Buffer{}

diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go
@@ -17,7 +17,6 @@ import (
 	"time"
 
 	"github.com/efficientgo/e2e"
-	e2edb "github.com/efficientgo/e2e/db"
 	"github.com/efficientgo/e2e/matchers"
 	"github.com/go-kit/log"
 	"github.com/oklog/ulid"
@@ -346,16 +345,12 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
 	testutil.Ok(t, os.MkdirAll(dir, os.ModePerm))
 
 	const bucket = "compact_test"
-	m := e2ethanos.NewMinio(e, "minio", bucket)
+	m, err := e2ethanos.NewMinio(e, "minio", bucket)
+	testutil.Ok(t, err)
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 
-	bkt, err := s3.NewBucketWithConfig(logger, s3.Config{
-		Bucket:    bucket,
-		AccessKey: e2edb.MinioAccessKey,
-		SecretKey: e2edb.MinioSecretKey,
-		Endpoint:  m.Endpoint("http"), // We need separate client config, when connecting to minio from outside.
-		Insecure:  true,
-	}, "test-feed")
+	bkt, err := s3.NewBucketWithConfig(logger,
+		e2ethanos.NewS3Config(bucket, m.Endpoint("https"), e.SharedDir()), "test-feed")
 	testutil.Ok(t, err)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
@@ -457,14 +452,8 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
 	}
 
 	svcConfig := client.BucketConfig{
-		Type: client.S3,
-		Config: s3.Config{
-			Bucket:    bucket,
-			AccessKey: e2edb.MinioAccessKey,
-			SecretKey: e2edb.MinioSecretKey,
-			Endpoint:  m.InternalEndpoint("http"),
-			Insecure:  true,
-		},
+		Type:   client.S3,
+		Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), e2ethanos.ContainerSharedDir),
 	}
 	str, err := e2ethanos.NewStoreGW(e, "1", svcConfig, "")
 	testutil.Ok(t, err)

diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go
@@ -4,9 +4,15 @@
 package e2ethanos
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"math/big"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -25,7 +31,9 @@ import (
 
 	"github.com/thanos-io/thanos/pkg/alert"
 	"github.com/thanos-io/thanos/pkg/httpconfig"
+	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/client"
+	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/queryfrontend"
 	"github.com/thanos-io/thanos/pkg/receive"
 )
@@ -814,37 +822,51 @@ http {
 // NewMinio returns minio server, used as a local replacement for S3.
 // TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11;
 // after this is addresses fixed all calls should be replaced with e2edb.NewMinio.
-func NewMinio(env e2e.Environment, name, bktName string) *e2e.InstrumentedRunnable {
+func NewMinio(env e2e.Environment, name, bktName string) (*e2e.InstrumentedRunnable, error) {
 	image := "minio/minio:RELEASE.2019-12-30T05-45-39Z"
 	minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master"
 	commands := []string{
 		"curl -sSL --tlsv1.2 -O '%s/root.key'	-O '%s/root.cert'",
-		"mkdir -p /data/%s && minio server --address :%v --quiet /data",
+		"mkdir -p /data/%s && minio server --certs-dir /shared/data/certs --address :%v --quiet /data",
+	}
+
+	if err := os.MkdirAll(filepath.Join(env.SharedDir(), "data", "certs", "CAs"), 0750); err != nil {
+		return nil, errors.Wrap(err, "create certs dir")
+	}
+
+	if err := genCerts(
+		filepath.Join(env.SharedDir(), "data", "certs", "public.crt"),
+		filepath.Join(env.SharedDir(), "data", "certs", "private.key"),
+		filepath.Join(env.SharedDir(), "data", "certs", "CAs", "ca.crt"),
+		env.Name()+"-"+name); err != nil {
+		return nil, errors.Wrap(err, "fail to generate certs")
 	}
 
 	return e2e.NewInstrumentedRunnable(
 		env,
 		name,
-		map[string]int{"http": 8090},
-		"http").Init(
+		map[string]int{"https": 8090},
+		"https").Init(
 		e2e.StartOptions{
 			Image: image,
 			// Create the required bucket before starting minio.
-			Command:   e2e.NewCommandWithoutEntrypoint("sh", "-c", fmt.Sprintf(strings.Join(commands, " && "), minioKESGithubContent, minioKESGithubContent, bktName, 8090)),
-			Readiness: e2e.NewHTTPReadinessProbe("http", "/minio/health/ready", 200, 200),
+			Command: e2e.NewCommandWithoutEntrypoint("sh", "-c", fmt.Sprintf(strings.Join(commands, " && "), minioKESGithubContent, minioKESGithubContent, bktName, 8090)),
+			//TODO(@clyang82): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/9
+			//Readiness: e2e.NewHTTPReadinessProbe("http", "/minio/health/ready", 200, 200),
+			Readiness: e2e.NewCmdReadinessProbe(e2e.NewCommand("sh", "-c", "sleep 1 && curl -k https://127.0.0.1:8090/minio/health/ready")),
 			EnvVars: map[string]string{
 				"MINIO_ACCESS_KEY": e2edb.MinioAccessKey,
 				"MINIO_SECRET_KEY": e2edb.MinioSecretKey,
 				"MINIO_BROWSER":    "off",
-				"ENABLE_HTTPS":     "0",
+				"ENABLE_HTTPS":     "1",
 				// https://docs.min.io/docs/minio-kms-quickstart-guide.html
 				"MINIO_KMS_KES_ENDPOINT":  "https://play.min.io:7373",
 				"MINIO_KMS_KES_KEY_FILE":  "root.key",
 				"MINIO_KMS_KES_CERT_FILE": "root.cert",
 				"MINIO_KMS_KES_KEY_NAME":  "my-minio-key",
 			},
 		},
-	)
+	), nil
 }
 
 func NewMemcached(e e2e.Environment, name string) *e2e.InstrumentedRunnable {
@@ -914,3 +936,90 @@ func NewToolsBucketWeb(
 
 	return toolsBucketWeb, nil
 }
+
+// genCerts generates certificates and writes those to the provided paths.
+func genCerts(certPath, privkeyPath, caPath, serverName string) error {
+	var caRoot = &x509.Certificate{
+		SerialNumber:          big.NewInt(2019),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		IsCA:                  true,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	var cert = &x509.Certificate{
+		SerialNumber: big.NewInt(1658),
+		DNSNames:     []string{serverName},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+		NotAfter:     time.Now().AddDate(10, 0, 0),
+		SubjectKeyId: []byte{1, 2, 3},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+	// Generate CA cert.
+	caBytes, err := x509.CreateCertificate(rand.Reader, caRoot, caRoot, &caPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		return err
+	}
+	caPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+	err = ioutil.WriteFile(caPath, caPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	// Sign the cert with the CA private key.
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caRoot, &certPrivKey.PublicKey, caPrivKey)
+	if err != nil {
+		return err
+	}
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	err = ioutil.WriteFile(certPath, certPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	err = ioutil.WriteFile(privkeyPath, certPrivKeyPEM, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewS3Config(bucket, endpoint, basePath string) s3.Config {
+	return s3.Config{
+		Bucket:    bucket,
+		AccessKey: e2edb.MinioAccessKey,
+		SecretKey: e2edb.MinioSecretKey,
+		Endpoint:  endpoint,
+		Insecure:  false,
+		HTTPConfig: s3.HTTPConfig{
+			TLSConfig: objstore.TLSConfig{
+				CAFile:   filepath.Join(basePath, "data", "certs", "CAs", "ca.crt"),
+				CertFile: filepath.Join(basePath, "data", "certs", "public.crt"),
+				KeyFile:  filepath.Join(basePath, "data", "certs", "private.key"),
+			},
+		},
+	}
+}
diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go
@@ -13,12 +13,10 @@ import (
 	"testing"
 	"time"
 
-	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
 	"github.com/efficientgo/e2e"
 	"github.com/prometheus/prometheus/model/labels"
 
 	"github.com/thanos-io/thanos/pkg/objstore/client"
-	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"github.com/thanos-io/thanos/pkg/query"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/pkg/testutil"
@@ -41,20 +39,15 @@ func TestInfo(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3))
 
 	const bucket = "info-api-test"
-	m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	m, err := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	testutil.Ok(t, err)
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 	store, err := e2ethanos.NewStoreGW(
 		e,
 		"1",
 		client.BucketConfig{
-			Type: client.S3,
-			Config: s3.Config{
-				Bucket:    bucket,
-				AccessKey: e2edb.MinioAccessKey,
-				SecretKey: e2edb.MinioSecretKey,
-				Endpoint:  m.InternalEndpoint("http"),
-				Insecure:  true,
-			},
+			Type:   client.S3,
+			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), e2ethanos.ContainerSharedDir),
 		},
 		"",
 	)

diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go
@@ -26,7 +26,6 @@ import (
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/chromedp"
 	"github.com/efficientgo/e2e"
-	e2edb "github.com/efficientgo/e2e/db"
 	"github.com/go-kit/log"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
@@ -655,7 +654,8 @@ func TestSidecarStorePushdown(t *testing.T) {
 	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1))
 
 	const bucket = "store_gateway_test"
-	m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	m, err := e2ethanos.NewMinio(e, "thanos-minio", bucket)
+	testutil.Ok(t, err)
 	testutil.Ok(t, e2e.StartAndWaitReady(m))
 
 	dir := filepath.Join(e.SharedDir(), "tmp")
@@ -672,13 +672,7 @@ func TestSidecarStorePushdown(t *testing.T) {
 	testutil.Ok(t, err)
 
 	l := log.NewLogfmtLogger(os.Stdout)
-	bkt, err := s3.NewBucketWithConfig(l, s3.Config{
-		Bucket:    bucket,
-		AccessKey: e2edb.MinioAccessKey,
-		SecretKey: e2edb.MinioSecretKey,
-		Endpoint:  m.Endpoint("http"),
-		Insecure:  true,
-	}, "test")
+	bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("https"), e.SharedDir()), "test")
 	testutil.Ok(t, err)
 
 	testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String()))
@@ -687,14 +681,8 @@ func TestSidecarStorePushdown(t *testing.T) {
 		e,
 		"1",
 		client.BucketConfig{
-			Type: client.S3,
-			Config: s3.Config{
-				Bucket:    bucket,
-				AccessKey: e2edb.MinioAccessKey,
-				SecretKey: e2edb.MinioSecretKey,
-				Endpoint:  m.InternalEndpoint("http"),
-				Insecure:  true,
-			},
+			Type:   client.S3,
+			Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), e2ethanos.ContainerSharedDir),
 		},
 		"",
 	)