[WIP] Remove the copy-cert init containers

cmd/self-signer/root.go

@@ -26,15 +26,17 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/cockroachdb/helm-charts/pkg/generator"
 )
 
 var (
-	cl  client.Client
-	ctx context.Context
+	cl     client.Client
+	config *rest.Config
+	ctx    context.Context
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -71,7 +73,7 @@ func init() {
 	runtimeScheme := runtime.NewScheme()
 
 	_ = clientgoscheme.AddToScheme(runtimeScheme)
-	config := controllerruntime.GetConfigOrDie()
+	config = controllerruntime.GetConfigOrDie()
 
 	cl, err = client.New(config, client.Options{
 		Scheme: runtimeScheme,
@@ -85,7 +87,7 @@ func init() {
 func getInitialConfig(caDuration, caExpiry, nodeDuration, nodeExpiry, clientDuration,
 	clientExpiry string) (generator.GenerateCert, error) {
 
-	genCert := generator.NewGenerateCert(cl)
+	genCert := generator.NewGenerateCert(config, cl)
 
 	if err := genCert.CaCertConfig.SetConfig(caDuration, caExpiry); err != nil {
 		return genCert, err

cockroachdb/templates/role-certRotateSelfSigner.yaml

@@ -22,6 +22,9 @@ rules:
     resourceNames:
       - {{ template "cockroachdb.fullname" . }}
   - apiGroups: [""]
-    resources: ["pods"]
+    resources: ["pods", "pods/logs"]
     verbs: ["delete", "get"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
 {{- end }}

cockroachdb/templates/statefulset.yaml

@@ -51,7 +51,7 @@ spec:
       {{- end }}
     {{- end }}
       serviceAccountName: {{ template "cockroachdb.serviceAccount.name" . }}
-      {{- if .Values.tls.enabled }}
+      {{- if and .Values.tls.enabled .Values.statefulset.securityContext.runAsNonRootUser.enabled }}
       initContainers:
         - name: copy-certs
           image: {{ .Values.tls.copyCerts.image | quote }}
@@ -266,11 +266,15 @@ spec:
             - name: datadir
               mountPath: /cockroach/{{ .Values.conf.path }}/
           {{- if .Values.tls.enabled }}
+            {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }}
             - name: certs
               mountPath: /cockroach/cockroach-certs/
-              {{- if .Values.tls.certs.provided }}
+            {{- end }}
             - name: certs-secret
+              {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }}
               mountPath: /cockroach/certs/
+              {{- else }}
+              mountPath: /cockroach/cockroach-certs/
               {{- end }}
           {{- end }}
           {{- range .Values.statefulset.secretMounts }}
@@ -340,34 +344,29 @@ spec:
           emptyDir: {}
         {{- end }}
       {{- if .Values.tls.enabled }}
+        {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled}}
         - name: certs
           emptyDir: {}
+        {{- end }}
           {{- if or .Values.tls.certs.provided .Values.tls.certs.certManager  .Values.tls.certs.selfSigner.enabled }}
         - name: certs-secret
-          {{- if or .Values.tls.certs.tlsSecret .Values.tls.certs.certManager .Values.tls.certs.selfSigner.enabled }}
-          projected:
-            sources:
-            - secret:
-                {{- if .Values.tls.certs.selfSigner.enabled }}
-                name: {{ template "cockroachdb.fullname" . }}-node-secret
-                {{ else }}
-                name: {{ .Values.tls.certs.nodeSecret }}
-                {{ end -}}
-                items:
-                - key: ca.crt
-                  path: ca.crt
-                  mode: 256
-                - key: tls.crt
-                  path: node.crt
-                  mode: 256
-                - key: tls.key
-                  path: node.key
-                  mode: 256
-          {{- else }}
           secret:
+            {{- if .Values.tls.certs.selfSigner.enabled }}
+            secretName: {{ template "cockroachdb.fullname" . }}-node-secret
+            {{ else }}
             secretName: {{ .Values.tls.certs.nodeSecret }}
+            {{- end }}
             defaultMode: 256
-          {{- end }}
+            items:
+              - key: ca.crt
+                path: ca.crt
+                mode: 256
+              - key: tls.crt
+                path: node.crt
+                mode: 256
+              - key: tls.key
+                path: node.key
+                mode: 256
           {{- end }}
       {{- end }}
       {{- range .Values.statefulset.secretMounts }}
@@ -385,10 +384,12 @@ spec:
       securityContext:
         seccompProfile:
           type: "RuntimeDefault"
+        {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }}
         fsGroup: 1000
         runAsGroup: 1000
         runAsUser: 1000
         runAsNonRoot: true
+        {{- end }}
       {{- end }}
       {{- end }}
 {{- if .Values.storage.persistentVolume.enabled }}

cockroachdb/values.yaml

@@ -283,6 +283,8 @@ statefulset:
 
   securityContext:
     enabled: true
+    runAsNonRootUser:
+      enabled: false
 
   serviceAccount:
     # Specifies whether this ServiceAccount should be created.
@@ -580,7 +582,7 @@ tls:
     # Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
     image:
       repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
-      tag: "1.5"
+      tag: "1.6"
       pullPolicy: IfNotPresent
       credentials: {}
       registry: gcr.io

pkg/generator/generate_cert.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/cockroachdb/helm-charts/pkg/kube"
@@ -52,6 +53,7 @@ func init() {
 // GenerateCert is the structure containing all the certificate related info
 type GenerateCert struct {
 	client                    client.Client
+	restConfig                *rest.Config
 	CertsDir                  string
 	CaSecret                  string
 	CAKey                     string
@@ -93,9 +95,10 @@ func (c *certConfig) SetConfig(duration, expiryWindow string) error {
 	return nil
 }
 
-func NewGenerateCert(cl client.Client) GenerateCert {
+func NewGenerateCert(config *rest.Config, cl client.Client) GenerateCert {
 	return GenerateCert{
 		client:           cl,
+		restConfig:       config,
 		CaCertConfig:     &certConfig{},
 		NodeCertConfig:   &certConfig{},
 		ClientCertConfig: &certConfig{},
@@ -370,7 +373,7 @@ func (rc *GenerateCert) generateNodeCert(ctx context.Context, nodeSecretName str
 					return err
 				}
 
-				if err = kube.RollingUpdate(ctx, rc.client, rc.DiscoveryServiceName, namespace, rc.ReadinessWait, rc.PodUpdateTimeout); err != nil {
+				if err = kube.SighupSignalToPods(ctx, rc.restConfig, rc.client, rc.DiscoveryServiceName, namespace); err != nil {
 					return
 				}
 				return nil

pkg/kube/helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kube
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"strconv"
@@ -29,6 +30,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -153,6 +158,72 @@ func RollingUpdate(ctx context.Context, cl client.Client, stsName, namespace str
 	return nil
 }
 
+// SighupSignalToPods sends SIGHUP signal to all the pods in the statefulset.
+func SighupSignalToPods(ctx context.Context, config *rest.Config, cl client.Client, stsName, namespace string) error {
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	var sts v1.StatefulSet
+	if err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: stsName}, &sts); err != nil {
+		return err
+	}
+
+	containerName := sts.Spec.Template.Spec.Containers[0].Name
+	command := []string{"/bin/bash", "-c", "echo 'Send SIGHUP to cockroach'; kill -s 1 $(ls -l /proc/*/exe | grep cockroach | awk '{print $2}')"}
+	for i := int32(0); i < sts.Status.Replicas; i++ {
+		replicaName := stsName + "-" + strconv.Itoa(int(i))
+
+		stdout, stderr, err := execCommandInPod(clientset, config, namespace, replicaName, containerName, command)
+		if err != nil {
+			logrus.Errorf("Failed to send SIGHUP signal to pod [%s], error: %v, stdout: %s, stderr: %s", replicaName, err, stdout, stderr)
+		}
+		logrus.Info(stdout)
+
+		// Sleeping for 1 second to allow the pod to receive the signal
+		time.Sleep(1 * time.Second)
+	}
+
+	return nil
+}
+
+// execCommandInPod executes the provided command in the given pod and returns the stdout and stderr.
+func execCommandInPod(clientset *kubernetes.Clientset, config *rest.Config, namespace, podName, containerName string, command []string) (string, string, error) {
+	logrus.Infof("Running command %s in pod %s in container %s", command, podName, containerName)
+
+	req := clientset.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(podName).
+		Namespace(namespace).
+		SubResource("exec").
+		Param("container", containerName).
+		VersionedParams(&corev1.PodExecOptions{
+			Command: command,
+			Stdin:   false,
+			Stdout:  true,
+			Stderr:  true,
+			TTY:     true,
+		}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
+	if err != nil {
+		return "", "", err
+	}
+
+	var stdout, stderr bytes.Buffer
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdout: &stdout,
+		Stderr: &stderr,
+	})
+	if err != nil {
+		return stdout.String(), stderr.String(), err
+	}
+
+	return stdout.String(), stderr.String(), nil
+}
+
 func WaitForPodReady(ctx context.Context, cl client.Client, name, namespace string, podUpdateTimeout,
 	podMaxPollingInterval time.Duration) error {
 	f := func() error {