diff --git a/deploy/charts/csi-driver-spiffe/README.md b/deploy/charts/csi-driver-spiffe/README.md index ed2d210..28807db 100644 --- a/deploy/charts/csi-driver-spiffe/README.md +++ b/deploy/charts/csi-driver-spiffe/README.md @@ -94,6 +94,15 @@ Verbosity of cert-manager-csi-driver logging. > ``` Duration requested for requested certificates. +#### **app.runtimeIssuanceConfigMap** ~ `string` +> Default value: +> ```yaml +> "" +> ``` + +Name of a ConfigMap in the installation namespace to watch, providing runtime configuration of an issuer to use. + +The "issuer.name", "issuer.kind" and "issuer.group" keys must be present in the ConfigMap for it to be used. #### **app.extraCertificateRequestAnnotations** ~ `unknown` > Default value: > ```yaml diff --git a/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml b/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml index eeac4a8..d378c1c 100644 --- a/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml @@ -83,6 +83,8 @@ spec: - --node-id=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --data-root=csi-data-dir + - "--runtime-issuance-config-map-name={{.Values.app.runtimeIssuanceConfigMap}}" + - "--runtime-issuance-config-map-namespace={{.Release.Namespace}}" {{- if .Values.app.extraCertificateRequestAnnotations }} - --extra-certificate-request-annotations={{ .Values.app.extraCertificateRequestAnnotations }} {{- end }} diff --git a/deploy/charts/csi-driver-spiffe/templates/role.yaml b/deploy/charts/csi-driver-spiffe/templates/role.yaml index 8df5440..4c00dd3 100644 --- a/deploy/charts/csi-driver-spiffe/templates/role.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/role.yaml @@ -1,3 +1,21 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cert-manager-csi-driver-spiffe.labels" . | nindent 4 }} +rules: +{{- if .Values.app.runtimeIssuanceConfigMap }} +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + resourceNames: ["{{.Values.app.runtimeIssuanceConfigMap}}"] +{{- end }} + + +--- + kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml b/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml index c5f5f0f..8e1a0d0 100644 --- a/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml @@ -1,3 +1,21 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cert-manager-csi-driver-spiffe.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + +--- + kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/charts/csi-driver-spiffe/values.schema.json b/deploy/charts/csi-driver-spiffe/values.schema.json index 354ba19..54b24ae 100644 --- a/deploy/charts/csi-driver-spiffe/values.schema.json +++ b/deploy/charts/csi-driver-spiffe/values.schema.json @@ -48,6 +48,9 @@ "name": { "$ref": "#/$defs/helm-values.app.name" }, + "runtimeIssuanceConfigMap": { + "$ref": "#/$defs/helm-values.app.runtimeIssuanceConfigMap" + }, "trustDomain": { "$ref": "#/$defs/helm-values.app.trustDomain" } @@ -430,6 +433,11 @@ "description": "The name for the CSI driver installation.", "type": "string" }, + "helm-values.app.runtimeIssuanceConfigMap": { + "default": "", + "description": "Name of a ConfigMap in the installation namespace to watch, providing runtime configuration of an issuer to use.\n\nThe \"issuer.name\", \"issuer.kind\" and \"issuer.group\" keys must be present in the ConfigMap for it to be used.", + "type": "string" + }, "helm-values.app.trustDomain": { "default": "cluster.local", "description": "The Trust Domain for this driver.", diff --git a/deploy/charts/csi-driver-spiffe/values.yaml b/deploy/charts/csi-driver-spiffe/values.yaml index fee004c..59caaf7 100644 --- a/deploy/charts/csi-driver-spiffe/values.yaml +++ b/deploy/charts/csi-driver-spiffe/values.yaml @@ -25,7 +25,7 @@ image: # driver: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 # +docs:property=image.digest.driver # driver: sha256:... - + # Target csi-driver approver digest. Override any tag, if set. # For example: # approver: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 @@ -47,6 +47,14 @@ app: logLevel: 1 # 1-5 # Duration requested for requested certificates. certificateRequestDuration: 1h + + # Name of a ConfigMap in the installation namespace to watch, providing + # runtime configuration of an issuer to use. + # + # The "issuer.name", "issuer.kind" and "issuer.group" keys must be present in + # the ConfigMap for it to be used. + runtimeIssuanceConfigMap: "" + # List of annotations to add to certificate requests # # For example: @@ -192,7 +200,7 @@ app: # Create Prometheus ServiceMonitor resource for cert-manager-csi-driver-spiffe approver. enabled: false # The value for the "prometheus" label on the ServiceMonitor. This allows - # for multiple Prometheus instances selecting difference ServiceMonitors + # for multiple Prometheus instances selecting difference ServiceMonitors # using label selectors. prometheusInstance: default # The interval that the Prometheus will scrape for metrics. diff --git a/internal/csi/app/app.go b/internal/csi/app/app.go index 5de4580..a51d259 100644 --- a/internal/csi/app/app.go +++ b/internal/csi/app/app.go @@ -71,7 +71,10 @@ func NewCommand(ctx context.Context) *cobra.Command { TrustDomain: opts.CertManager.TrustDomain, CertificateRequestAnnotations: opts.CertManager.CertificateRequestAnnotations, CertificateRequestDuration: opts.CertManager.CertificateRequestDuration, - IssuerRef: opts.CertManager.IssuerRef, + IssuerRef: &opts.CertManager.IssuerRef, + + IssuanceConfigMapName: opts.CertManager.IssuanceConfigMapName, + IssuanceConfigMapNamespace: opts.CertManager.IssuanceConfigMapNamespace, CertificateFileName: opts.Volume.CertificateFileName, KeyFileName: opts.Volume.KeyFileName, diff --git a/internal/csi/app/options/options.go b/internal/csi/app/options/options.go index 1c83161..1bdfa18 100644 --- a/internal/csi/app/options/options.go +++ b/internal/csi/app/options/options.go @@ -56,6 +56,12 @@ type OptionsDriver struct { // OptionsCertManager is options specific to cert-manager CertificateRequests. type OptionsCertManager struct { + // IssuanceConfigMapName is the name of a ConfigMap to watch for configuration options. The ConfigMap is expected to be in the same namespace as the csi-driver-spiffe pod. + IssuanceConfigMapName string + + // IssuanceConfigMapNamespace is the namespace where the runtime configuration ConfigMap is located + IssuanceConfigMapNamespace string + // TrustDomain is the trust domain of this SPIFFE PKI. The TrustDomain will // appear in signed certificate's URI SANs. TrustDomain string @@ -113,6 +119,10 @@ func (o *Options) addDriverFlags(fs *pflag.FlagSet) { } func (o *Options) addCertManagerFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.CertManager.IssuanceConfigMapName, "runtime-issuance-config-map-name", "", "Name of a ConfigMap to watch at runtime for issuer details. If such a ConfigMap is found, overrides issuer-name, issuer-kind and issuer-group") + + fs.StringVar(&o.CertManager.IssuanceConfigMapNamespace, "runtime-issuance-config-map-namespace", "", "Namespace for ConfigMap to be watched at runtime for issuer details") + fs.StringVar(&o.CertManager.TrustDomain, "trust-domain", "cluster.local", "The trust domain that will be requested for on created CertificateRequests.") fs.DurationVar(&o.CertManager.CertificateRequestDuration, "certificate-request-duration", time.Hour, diff --git a/internal/csi/driver/driver.go b/internal/csi/driver/driver.go index 20336c4..c411fa8 100644 --- a/internal/csi/driver/driver.go +++ b/internal/csi/driver/driver.go @@ -38,8 +38,12 @@ import ( "github.com/cert-manager/csi-lib/storage" "github.com/go-logr/logr" "gopkg.in/square/go-jose.v2/jwt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cert-manager/csi-driver-spiffe/internal/annotations" "github.com/cert-manager/csi-driver-spiffe/internal/csi/rootca" @@ -73,7 +77,7 @@ type Options struct { CertificateRequestDuration time.Duration // IssuerRef is the IssuerRef used when creating CertificateRequests. - IssuerRef cmmeta.ObjectReference + IssuerRef *cmmeta.ObjectReference // CertificateFileName is the name of the file that the signed certificate // will be written to inside the Pod's volume. @@ -98,6 +102,12 @@ type Options struct { // CAFileName. If the root CA certificate data changes, all managed volume's // file will be updated. RootCAs rootca.Interface + + // IssuanceConfigMapName is the name of the ConfigMap to watch for issuance configuration. + IssuanceConfigMapName string + + // IssuanceConfigMapNamespace is the namespace of the ConfigMap to watch for issuance configuration + IssuanceConfigMapNamespace string } // Driver is used for running the actual CSI driver. Driver will respond to @@ -117,9 +127,20 @@ type Driver struct { // created CertificateRequests. certificateRequestDuration time.Duration - // issuerRef is the issuerRef that will be set on all created - // CertificateRequests. - issuerRef cmmeta.ObjectReference + // activeIssuerRef is the issuerRef that will be set on all created CertificateRequests. + // Can be changed at runtime via runtime configuration (i.e. reading from a ConfigMap) + // Not to be confused with originalIssuerRef, which is an issuerRef optionally passed in + // via CLI args. + activeIssuerRef *cmmeta.ObjectReference + + // originalIssuerRef is the issuerRef passed into the driver at startup. This will be used + // if no runtime configuration (ConfigMap configuration) is found, or if the ConfigMap for + // runtime configuration is deleted. + originalIssuerRef *cmmeta.ObjectReference + + // activeIssuerRefMutex is used to control changes to the activeIssuerRef which can happen + // concurrently with a request to issue a new cert + activeIssuerRefMutex *sync.RWMutex // certFileName, keyFileName, caFileName are the names used when writing file // to volumes. @@ -138,6 +159,17 @@ type Driver struct { // camanager is used to update all managed volumes with the current root CA // certificates PEM. camanager *camanager + + // kubernetesClient is used to watch ConfigMaps for issuance configuration + kubernetesClient client.WithWatch + + // issuanceConfigMapName is the name of a ConfigMap which will be + // watched for issuance configuration at runtime + issuanceConfigMapName string + + // issuanceConfigMapNamespace is the name of a ConfigMap which will be + // watched for issuance configuration at runtime + issuanceConfigMapNamespace string } // New constructs a new Driver instance. @@ -148,16 +180,39 @@ func New(log logr.Logger, opts Options) (*Driver, error) { // don't exit, not a fatal error as sanitizeAnnotations will trim bad annotations } + originalIssuerRef, err := handleOriginalIssuerRef(opts.IssuerRef) + if err != nil && err != errNoOriginalIssuer { + return nil, err + } + + if originalIssuerRef == nil && (opts.IssuanceConfigMapName == "" || opts.IssuanceConfigMapNamespace == "") { + // if no install-time issuer was configured, runtime issuance details are not optional + return nil, fmt.Errorf("runtime issuance configuration is required if no issuer is provided at startup") + } + d := &Driver{ log: log.WithName("csi"), trustDomain: opts.TrustDomain, certFileName: opts.CertificateFileName, keyFileName: opts.KeyFileName, - issuerRef: opts.IssuerRef, - rootCAs: opts.RootCAs, + + // we check if we can set activeIssuerRef later + activeIssuerRef: nil, + originalIssuerRef: originalIssuerRef, + + activeIssuerRefMutex: &sync.RWMutex{}, + + rootCAs: opts.RootCAs, certificateRequestDuration: opts.CertificateRequestDuration, certificateRequestAnnotations: sanitizedAnnotations, + + issuanceConfigMapName: opts.IssuanceConfigMapName, + issuanceConfigMapNamespace: opts.IssuanceConfigMapNamespace, + } + + if d.originalIssuerRef != nil { + d.activeIssuerRef = d.originalIssuerRef } if len(d.certFileName) == 0 { @@ -194,6 +249,13 @@ func New(log logr.Logger, opts Options) (*Driver, error) { return nil, fmt.Errorf("failed to build cert-manager client: %w", err) } + k8sClient, err := client.NewWithWatch(opts.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to build kubernetes watcher client: %w", err) + } + + d.kubernetesClient = k8sClient + mngrLog := d.log.WithName("manager") d.driver, err = driver.New(opts.Endpoint, d.log.WithName("driver"), driver.Options{ DriverName: opts.DriverName, @@ -222,7 +284,150 @@ func New(log logr.Logger, opts Options) (*Driver, error) { return d, nil } -// Run is a blocking func that run the CSI driver. +// watchRuntimeConfigurationSource should be called in a goroutine to watch a ConfigMap for runtime configuration +func (d *Driver) watchRuntimeConfigurationSource(ctx context.Context) { + logger := d.log.WithName("runtime-config-watcher").WithValues("config-map-name", d.issuanceConfigMapName, "config-map-namespace", d.issuanceConfigMapNamespace) + +LOOP: + for { + logger.Info("Starting / restarting watcher for runtime configuration") + cmList := &corev1.ConfigMapList{} + + // First create a watcher. This is in a labelled loop in case the watcher dies for some reason + // while we're running - in that case, we don't want to give up entirely on watching for runtime config + // but instead we want to recreate the watcher. + + watcher, err := d.kubernetesClient.Watch(ctx, cmList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", d.issuanceConfigMapName), + Namespace: d.issuanceConfigMapNamespace, + }) + + if err != nil { + logger.Error(err, "Failed to create ConfigMap watcher; will retry in 5s") + time.Sleep(5 * time.Second) + continue + } + + for { + // Now loop indefinitely until the main context cancels or we get an event to process. + // If the main context cancels, we break out of the outer loop and this function returns. + // If we get an event, we first check whether the channel closed. If so, we recreate the watcher by continuing + // the outer loop. + select { + case <-ctx.Done(): + logger.Info("Received context cancellation, shutting down runtime configuration watcher") + watcher.Stop() + break LOOP + + case event, open := <-watcher.ResultChan(): + if !open { + logger.Info("Received closed channel from ConfigMap watcher, will recreate") + watcher.Stop() + continue LOOP + } + + switch event.Type { + case watch.Deleted: + d.handleRuntimeConfigIssuerDeletion(logger) + + case watch.Added: + err := d.handleRuntimeConfigIssuerChange(logger, event) + if err != nil { + logger.Error(err, "Failed to handle new runtime configuration for issuerRef") + } + + case watch.Modified: + err := d.handleRuntimeConfigIssuerChange(logger, event) + if err != nil { + logger.Error(err, "Failed to handle runtime configuration issuerRef change") + } + + case watch.Bookmark: + // Ignore + + case watch.Error: + err, ok := event.Object.(error) + if !ok { + logger.Error(nil, "Got an error event when watching runtime configuration but unable to determine further information") + } else { + logger.Error(err, "Got an error event when watching runtime configuration") + } + + default: + logger.Info("Got unknown event for runtime configuration ConfigMap; ignoring", "event-type", string(event.Type)) + } + } + } + } + + logger.Info("Stopped runtime configuration watcher") +} + +const ( + issuerNameKey = "issuer.name" + issuerKindKey = "issuer.kind" + issuerGroupKey = "issuer.group" +) + +func (d *Driver) handleRuntimeConfigIssuerChange(logger logr.Logger, event watch.Event) error { + d.activeIssuerRefMutex.Lock() + defer d.activeIssuerRefMutex.Unlock() + + cm, ok := event.Object.(*corev1.ConfigMap) + if !ok { + return fmt.Errorf("got unexpected type for runtime configuration source; this is likely a programming error") + } + + issuerRef := &cmmeta.ObjectReference{} + + var dataErrs []error + var exists bool + + issuerRef.Name, exists = cm.Data[issuerNameKey] + if !exists || len(issuerRef.Name) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data: %s", issuerNameKey)) + } + + issuerRef.Kind, exists = cm.Data[issuerKindKey] + if !exists || len(issuerRef.Kind) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data: %s", issuerKindKey)) + } + + issuerRef.Group, exists = cm.Data[issuerGroupKey] + if !exists || len(issuerRef.Group) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data; %s", issuerGroupKey)) + } + + if len(dataErrs) > 0 { + return errors.Join(dataErrs...) + } + + // we now have a full issuerRef + // TODO: check if the issuer exists by querying for the CRD? + + d.activeIssuerRef = issuerRef + + logger.Info("Changed active issuerRef in response to runtime configuration ConfigMap", "issuer-name", d.activeIssuerRef.Name, "issuer-kind", d.activeIssuerRef.Kind, "issuer-group", d.activeIssuerRef.Group) + + return nil +} + +func (d *Driver) handleRuntimeConfigIssuerDeletion(logger logr.Logger) { + d.activeIssuerRefMutex.Lock() + defer d.activeIssuerRefMutex.Unlock() + + if d.originalIssuerRef == nil { + logger.Info("Runtime issuance configuration was deleted and no issuerRef was configured at install time; issuance will fail until runtime configuration is reinstated") + d.activeIssuerRef = nil + return + } + + logger.Info("Runtime issuance configuration was deleted; issuance will revert to original issuerRef configured at install time") + + d.activeIssuerRef = d.originalIssuerRef +} + +// Run is a blocking func that runs the CSI driver. func (d *Driver) Run(ctx context.Context) error { var wg sync.WaitGroup @@ -237,6 +442,12 @@ func (d *Driver) Run(ctx context.Context) error { d.camanager.run(ctx, time.Second*5) }() + wg.Add(1) + go func() { + defer wg.Done() + d.watchRuntimeConfigurationSource(ctx) + }() + wg.Add(1) var err error go func() { @@ -251,6 +462,13 @@ func (d *Driver) Run(ctx context.Context) error { // generateRequest will generate a SPIFFE manager.CertificateRequestBundle // based upon the identity contained in the metadata service account token. func (d *Driver) generateRequest(meta metadata.Metadata) (*manager.CertificateRequestBundle, error) { + d.activeIssuerRefMutex.RLock() + defer d.activeIssuerRefMutex.RUnlock() + + if d.activeIssuerRef == nil { + return nil, fmt.Errorf("no issuerRef is currently active for csi-driver-spiffe; configure one using runtime configuration") + } + // Extract the service account token from the volume metadata in order to // derive the service account, and thus identity of the pod. token, err := util.EmptyAudienceTokenFromMetadata(meta) @@ -311,7 +529,7 @@ func (d *Driver) generateRequest(meta metadata.Metadata) (*manager.CertificateRe cmapi.UsageServerAuth, cmapi.UsageClientAuth, }, - IssuerRef: d.issuerRef, + IssuerRef: *d.activeIssuerRef, Annotations: crAnnotations, }, nil } @@ -376,3 +594,29 @@ func sanitizeAnnotations(in map[string]string) (map[string]string, error) { return out, errors.Join(errs...) } + +var errNoOriginalIssuer = fmt.Errorf("no original issuer was provided") + +func handleOriginalIssuerRef(in *cmmeta.ObjectReference) (*cmmeta.ObjectReference, error) { + if in == nil { + return nil, errNoOriginalIssuer + } + + if in.Name == "" && in.Kind == "" && in.Group == "" { + return nil, errNoOriginalIssuer + } + + if in.Name == "" { + return nil, fmt.Errorf("issuerRef.Name is a required field if any field is set for issuerRef") + } + + if in.Kind == "" { + return nil, fmt.Errorf("issuerRef.Kind is a required field if any field is set for issuerRef") + } + + if in.Group == "" { + return nil, fmt.Errorf("issuerRef.Group is a required field if any field is set for issuerRef") + } + + return in, nil +} diff --git a/make/test-e2e.mk b/make/test-e2e.mk index 5439a50..3eb1b8b 100644 --- a/make/test-e2e.mk +++ b/make/test-e2e.mk @@ -54,21 +54,23 @@ ifeq ($(findstring test-e2e,$(MAKECMDGOALS)),test-e2e) install: e2e-setup-example kind-cluster oci-load-manager oci-load-approver endif +E2E_RUNTIME_CONFIG_MAP_NAME ?= runtime-config-map +E2E_FOCUS ?= + test-e2e-deps: INSTALL_OPTIONS := test-e2e-deps: INSTALL_OPTIONS += --set image.repository.driver=$(oci_manager_image_name_development) test-e2e-deps: INSTALL_OPTIONS += --set image.repository.approver=$(oci_approver_image_name_development) test-e2e-deps: INSTALL_OPTIONS += --set image.pullPolicy=Never test-e2e-deps: INSTALL_OPTIONS += --set app.trustDomain=foo.bar -test-e2e-deps: INSTALL_OPTIONS += --set app.approver.signerName=clusterissuers.cert-manager.io/csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.issuer.name=csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumes[0].name=root-cas test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumes[0].secret.secretName=csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumeMounts[0].name=root-cas test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumeMounts[0].mountPath=/var/run/secrets/cert-manager-csi-driver-spiffe test-e2e-deps: INSTALL_OPTIONS += --set app.driver.sourceCABundle=/var/run/secrets/cert-manager-csi-driver-spiffe/ca.crt +test-e2e-deps: INSTALL_OPTIONS += --set app.runtimeIssuanceConfigMap=$(E2E_RUNTIME_CONFIG_MAP_NAME) test-e2e-deps: install -E2E_FOCUS ?= .PHONY: test-e2e ## e2e end-to-end tests @@ -82,4 +84,5 @@ test-e2e: test-e2e-deps | kind-cluster $(NEEDS_GINKGO) $(NEEDS_KUBECTL) $(ARTIFA -ldflags $(go_manager_ldflags) \ -- \ --kubeconfig-path=$(CURDIR)/$(kind_kubeconfig) \ - --kubectl-path=$(KUBECTL) + --kubectl-path=$(KUBECTL) \ + --runtime-issuance-config-map-name=$(E2E_RUNTIME_CONFIG_MAP_NAME) diff --git a/test/e2e/framework/config/config.go b/test/e2e/framework/config/config.go index d798a20..63b1c83 100644 --- a/test/e2e/framework/config/config.go +++ b/test/e2e/framework/config/config.go @@ -48,6 +48,9 @@ type Config struct { IssuerSecretName string RestConfig *rest.Config KubectlBinPath string + + IssuanceConfigMapName string + IssuanceConfigMapNamespace string } func (c *Config) AddFlags(fs *flag.FlagSet) *Config { @@ -89,5 +92,7 @@ func (c *Config) addFlags(fs *flag.FlagSet) *Config { fs.StringVar(&c.IssuerRef.Group, "issuer-group", "cert-manager.io", "Group of issuer which has been created for the test") fs.StringVar(&c.IssuerSecretName, "issuer-secret-name", "csi-driver-spiffe-ca", "Name of the CA certificate Secret") fs.StringVar(&c.IssuerSecretNamespace, "issuer-secret-namespace", "cert-manager", "Namespace where the CA certificate Secret is stored") + fs.StringVar(&c.IssuanceConfigMapName, "runtime-issuance-config-map-name", "runtime-config-map", "Name of runtime issuance ConfigMap") + fs.StringVar(&c.IssuanceConfigMapNamespace, "runtime-issuance-config-map-namespace", "cert-manager", "Namespace for runtime issuance ConfigMap") return c } diff --git a/test/e2e/suite/import.go b/test/e2e/suite/import.go index 8638cda..cc8760a 100644 --- a/test/e2e/suite/import.go +++ b/test/e2e/suite/import.go @@ -20,4 +20,5 @@ import ( _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/approval" _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/carotation" _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/fsgroup" + _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/runtimeconfiguration" ) diff --git a/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go b/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go new file mode 100644 index 0000000..bd369de --- /dev/null +++ b/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go @@ -0,0 +1,369 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeconfiguration + +import ( + "time" + + "github.com/cert-manager/cert-manager/pkg/util/pki" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/cert-manager/csi-driver-spiffe/test/e2e/framework" + "github.com/cert-manager/csi-driver-spiffe/test/e2e/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + mountPath = "/var/run/secrets/my-pod" + containerName = "my-container" +) + +var _ = framework.CasesDescribe("RuntimeConfiguration", func() { + f := framework.NewDefaultFramework("RuntimeConfiguration") + + var ( + serviceAccount corev1.ServiceAccount + role rbacv1.Role + rolebinding rbacv1.RoleBinding + podTemplate corev1.Pod + ) + + JustBeforeEach(func() { + By("Creating test resources") + + serviceAccount = corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Namespace: f.Namespace.Name, GenerateName: "csi-driver-spiffe-e2e-sa-"}, + } + Expect(f.Client().Create(f.Context(), &serviceAccount)).NotTo(HaveOccurred()) + + role = rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "csi-driver-spiffe-e2e-role-", + Namespace: f.Namespace.Name, + }, + Rules: []rbacv1.PolicyRule{{ + Verbs: []string{"create"}, + APIGroups: []string{"cert-manager.io"}, + Resources: []string{"certificaterequests"}, + }}, + } + Expect(f.Client().Create(f.Context(), &role)).NotTo(HaveOccurred()) + + rolebinding = rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "csi-driver-spiffe-e2e-rolebinding-", + Namespace: f.Namespace.Name, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccount.Name, + Namespace: f.Namespace.Name, + }}, + } + Expect(f.Client().Create(f.Context(), &rolebinding)).NotTo(HaveOccurred()) + + podTemplate = corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-pod-", + Namespace: f.Namespace.Name, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccount.Name, + Volumes: []corev1.Volume{{ + Name: "csi-driver-spiffe", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "spiffe.csi.cert-manager.io", + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{}, + }, + }, + }}, + Containers: []corev1.Container{ + { + Name: containerName, + Image: "docker.io/library/busybox:1.36.1-musl", + ImagePullPolicy: corev1.PullNever, + Command: []string{"sleep", "10000"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "csi-driver-spiffe", + MountPath: mountPath, + }, + }, + }, + }, + }, + } + + }) + + JustAfterEach(func() { + By("Cleaning up test resources") + Expect(f.Client().Delete(f.Context(), &rolebinding)).NotTo(HaveOccurred()) + Expect(f.Client().Delete(f.Context(), &role)).NotTo(HaveOccurred()) + Expect(f.Client().Delete(f.Context(), &serviceAccount)).NotTo(HaveOccurred()) + }) + + It("should succeed with a simple pod and no runtime configuration", func() { + pod := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &pod)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &pod)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &pod)).NotTo(HaveOccurred()) + + bundle, err := util.ReadCertFromMountPath(f, mountPath, pod.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + Expect(bundle.CheckNotEmpty()).NotTo(HaveOccurred()) + }) + + It("should succeed with a new issuer configured at runtime and revert when runtime configuration is deleted", func() { + // podOne should be created with the old issuer, since no ConfigMap has been created yet + By("Creating a pod before any runtime configuration") + podOne := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podOne)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podOne)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podOne)).NotTo(HaveOccurred()) + + By("Checking the pod used the issuer configured on startup") + cliArgCertBundle, err := util.ReadCertFromSecret(f, f.Config().IssuerSecretName, f.Config().IssuerSecretNamespace) + Expect(err).NotTo(HaveOccurred()) + + cliArgCert, err := pki.DecodeX509CertificateBytes(cliArgCertBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + podOneBundle, err := util.ReadCertFromMountPath(f, mountPath, podOne.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podOneChain, err := pki.DecodeX509CertificateChainBytes(podOneBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podOneChain[0].CheckSignatureFrom(cliArgCert)).NotTo(HaveOccurred()) + + By("Creating a new issuer to use at runtime") + issuerRef, newCABundle, cleanup, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanup() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACert, err := pki.DecodeX509CertificateBytes(newCABundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Config().IssuanceConfigMapName, + Namespace: f.Config().IssuanceConfigMapNamespace, + }, + Data: map[string]string{ + "issuer.name": issuerRef.Name, + "issuer.kind": issuerRef.Kind, + "issuer.group": issuerRef.Group, + }, + } + + By("Creating runtime configuration to point at the new issuer") + Expect(f.Client().Create(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + configMapNeedsCleanup := true + defer func() { + if configMapNeedsCleanup { + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + } + }() + + By("Waiting a little for runtime configuration to propagate") + time.Sleep(5 * time.Second) + + // now we've created the runtime configuration configmap, a newly created pod should + // use the new issuer + + By("Creating a second pod after runtime configuration was created") + podTwo := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podTwo)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podTwo)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podTwo)).NotTo(HaveOccurred()) + + By("Checking that the second pod used the new issuer") + podTwoBundle, err := util.ReadCertFromMountPath(f, mountPath, podTwo.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podTwoChain, err := pki.DecodeX509CertificateChainBytes(podTwoBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podTwoChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACert)).NotTo(HaveOccurred()) + + By("Deleting the configuration ConfigMap") + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + // we explicitly deleted the ConfigMap as part of the test - no need to clean it up any more + configMapNeedsCleanup = false + + By("Waiting a little for runtime configuration to revert") + time.Sleep(5 * time.Second) + + By("Creating a third pod after runtime configuration was deleted") + podThree := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podThree)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podThree)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podThree)).NotTo(HaveOccurred()) + + By("Checking that the third pod used the original issuer") + podThreeBundle, err := util.ReadCertFromMountPath(f, mountPath, podThree.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podThreeChain, err := pki.DecodeX509CertificateChainBytes(podThreeBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podThreeChain[0].CheckSignatureFrom(newCACert)).To(HaveOccurred()) + Expect(podThreeChain[0].CheckSignatureFrom(cliArgCert)).NotTo(HaveOccurred()) + }) + + It("should succeed with a new issuer configured at runtime and change issuers when configuration is updated", func() { + // First, fetch the cert for the CLI arg, to check later that it wasn't used to sign any pod certificates + cliArgCertBundle, err := util.ReadCertFromSecret(f, f.Config().IssuerSecretName, f.Config().IssuerSecretNamespace) + Expect(err).NotTo(HaveOccurred()) + + cliArgCert, err := pki.DecodeX509CertificateBytes(cliArgCertBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a new issuer to use at runtime") + issuerRefOne, newCABundleOne, cleanupIssuerOne, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanupIssuerOne() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACertOne, err := pki.DecodeX509CertificateBytes(newCABundleOne.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Config().IssuanceConfigMapName, + Namespace: f.Config().IssuanceConfigMapNamespace, + }, + Data: map[string]string{ + "issuer.name": issuerRefOne.Name, + "issuer.kind": issuerRefOne.Kind, + "issuer.group": issuerRefOne.Group, + }, + } + + By("Creating runtime configuration to point at the new issuer") + Expect(f.Client().Create(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + defer func() { + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + }() + + By("Waiting a little for runtime configuration to propagate") + time.Sleep(5 * time.Second) + + By("Creating a pod which uses runtime configuration") + podOne := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podOne)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podOne)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podOne)).NotTo(HaveOccurred()) + + By("Checking the pod used the new issuer") + podOneBundle, err := util.ReadCertFromMountPath(f, mountPath, podOne.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podOneChain, err := pki.DecodeX509CertificateChainBytes(podOneBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podOneChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podOneChain[0].CheckSignatureFrom(newCACertOne)).NotTo(HaveOccurred()) + + By("Creating a second new issuer to use at runtime") + issuerRefTwo, newCABundleTwo, cleanupIssuerTwo, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanupIssuerTwo() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACertTwo, err := pki.DecodeX509CertificateBytes(newCABundleTwo.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap.Data["issuer.name"] = issuerRefTwo.Name + runtimeConfigMap.Data["issuer.kind"] = issuerRefTwo.Kind + runtimeConfigMap.Data["issuer.group"] = issuerRefTwo.Group + + By("Updating runtime configuration to point at the new issuer") + Expect(f.Client().Update(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + By("Waiting a little for the runtime configuration update to propagate") + time.Sleep(5 * time.Second) + + By("Creating a second pod after runtime configuration was updated") + podTwo := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podTwo)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podTwo)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podTwo)).NotTo(HaveOccurred()) + + By("Checking that the second pod used the new issuer") + podTwoBundle, err := util.ReadCertFromMountPath(f, mountPath, podTwo.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podTwoChain, err := pki.DecodeX509CertificateChainBytes(podTwoBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podTwoChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACertOne)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACertTwo)).NotTo(HaveOccurred()) + }) +})