feat: add certStoreManager interface to wrap operations on namespaced…

… certStores [multi-tenancy PR 5] (#1382)

docs/design/Azure Kubernetes Workload Identity AuthProvider.md

@@ -19,7 +19,7 @@ User would need to follow instructions [here](https://azure.github.io/azure-work
 The official steps for setting up Workload Identity on AKS can be found [here](https://azure.github.io/azure-workload-identity/docs/quick-start.html).  
 
 1. Create ACR
-2. Create OIDC enabled AKS cluster by follow steps [here](https://docs.microsoft.com/en-us/azure/aks/cluster-configuration#oidc-issuer-preview)
+2. Create OIDC enabled AKS cluster by follow steps [here](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer)
 3. Save the cluster's OIDC URL: `az aks show --resource-group <resource_group> --name <cluster_name> --query "oidcIssuerProfile.issuerUrl" -otsv`
 4. Install Mutating Admission Webhook onto AKS cluster by following steps [here](https://azure.github.io/azure-workload-identity/docs/installation/mutating-admission-webhook.html)
 5. As the guide linked above shows, it's possible to use the AZ workload identity CLI or the regular az CLI to perform remaining setup. Following steps follow the AZ CLI.

pkg/controllers/certificatestore_controller.go

@@ -15,11 +15,11 @@ package controllers
 
 import (
 	"context"
-	"crypto/x509"
 	"encoding/json"
 	"fmt"
 
 	configv1beta1 "github.com/deislabs/ratify/api/v1beta1"
+	"github.com/deislabs/ratify/internal/constants"
 	"github.com/deislabs/ratify/pkg/certificateprovider"
 	_ "github.com/deislabs/ratify/pkg/certificateprovider/azurekeyvault" // register azure keyvault certificate provider
 	_ "github.com/deislabs/ratify/pkg/certificateprovider/inline"        // register inline certificate provider
@@ -40,11 +40,6 @@ type CertificateStoreReconciler struct {
 	Scheme *runtime.Scheme
 }
 
-var (
-	// a map between CertificateStore name to array of x509 certificates
-	certificatesMap = map[string][]*x509.Certificate{}
-)
-
 //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=certificatestores,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=certificatestores/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=certificatestores/finalizers,verbs=update
@@ -68,7 +63,8 @@ func (r *CertificateStoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	if err := r.Get(ctx, req.NamespacedName, &certStore); err != nil {
 		if apierrors.IsNotFound(err) {
 			logger.Infof("deletion detected, removing certificate store %v", resource)
-			delete(certificatesMap, resource)
+			// TODO: pass the actual namespace once multi-tenancy is supported.
+			NamespacedCertStores.DeleteStore(constants.EmptyNamespace, resource)
 		} else {
 			logger.Error(err, "unable to fetch certificate store")
 		}
@@ -99,7 +95,8 @@ func (r *CertificateStoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, fmt.Errorf("Error fetching certificates in store %v with %v provider, error: %w", resource, certStore.Spec.Provider, err)
 	}
 
-	certificatesMap[resource] = certificates
+	// TODO: pass the actual namespace once multi-tenancy is supported.
+	NamespacedCertStores.AddStore(constants.EmptyNamespace, resource, certificates)
 	isFetchSuccessful = true
 	emptyErrorString := ""
 	writeCertStoreStatus(ctx, r, certStore, logger, isFetchSuccessful, emptyErrorString, lastFetchedTime, certAttributes)
@@ -110,11 +107,6 @@ func (r *CertificateStoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	return ctrl.Result{}, nil
 }
 
-// returns the internal certificate map
-func GetCertificatesMap() map[string][]*x509.Certificate {
-	return certificatesMap
-}
-
 // SetupWithManager sets up the controller with the Manager.
 func (r *CertificateStoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	pred := predicate.GenerationChangedPredicate{}

pkg/controllers/policy_controller.go

@@ -58,7 +58,7 @@ func (r *PolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		if apierrors.IsNotFound(err) {
 			policyLogger.Infof("delete event detected, removing policy %s", resource)
 			// TODO: pass the actual namespace once multi-tenancy is supported.
-			ActivePolicies.DeletePolicy(constants.EmptyNamespace, resource)
+			NamespacedPolicies.DeletePolicy(constants.EmptyNamespace, resource)
 		} else {
 			policyLogger.Error("failed to get Policy: ", err)
 		}
@@ -96,7 +96,7 @@ func policyAddOrReplace(spec configv1beta1.PolicySpec) error {
 	}
 
 	// TODO: pass the actual namespace once multi-tenancy is supported.
-	ActivePolicies.AddPolicy(constants.EmptyNamespace, constants.RatifyPolicy, policyEnforcer)
+	NamespacedPolicies.AddPolicy(constants.EmptyNamespace, constants.RatifyPolicy, policyEnforcer)
 	return nil
 }
 

pkg/controllers/resource_map.go

@@ -14,18 +14,23 @@ limitations under the License.
 package controllers
 
 import (
+	cs "github.com/deislabs/ratify/pkg/customresources/certificatestores"
 	"github.com/deislabs/ratify/pkg/customresources/policies"
 	rs "github.com/deislabs/ratify/pkg/customresources/referrerstores"
 	"github.com/deislabs/ratify/pkg/customresources/verifiers"
 )
 
 var (
-	VerifierMap = verifiers.NewActiveVerifiers()
+	// NamespacedVerifiers is a map between namespace and verifiers.
+	NamespacedVerifiers = verifiers.NewActiveVerifiers()
 
-	// ActivePolicy is the active policy generated from CRD. There would be exactly
+	// NamespacedPolicies is the active policy generated from CRD. There would be exactly
 	// one active policy belonging to a namespace at any given time.
-	ActivePolicies = policies.NewActivePolicies()
+	NamespacedPolicies = policies.NewActivePolicies()
 
-	// a map to track active stores
-	StoreMap = rs.NewActiveStores()
+	// NamespacedStores is a map to track active stores across namespaces.
+	NamespacedStores = rs.NewActiveStores()
+
+	// NamespacedCertStores is a map between namespace and CertificateStores.
+	NamespacedCertStores = cs.NewActiveCertStores()
 )

pkg/controllers/store_controller.go

@@ -60,7 +60,7 @@ func (r *StoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		if apierrors.IsNotFound(err) {
 			storeLogger.Infof("deletion detected, removing store %v", req.Name)
 			// TODO: pass the actual namespace once multi-tenancy is supported.
-			StoreMap.DeleteStore(constants.EmptyNamespace, resource)
+			NamespacedStores.DeleteStore(constants.EmptyNamespace, resource)
 		} else {
 			storeLogger.Error(err, "unable to fetch store")
 		}
@@ -112,7 +112,7 @@ func storeAddOrReplace(spec configv1beta1.StoreSpec, fullname string) error {
 	}
 
 	// TODO: pass the actual namespace once multi-tenancy is supported.
-	StoreMap.AddStore(constants.EmptyNamespace, fullname, storeReference)
+	NamespacedStores.AddStore(constants.EmptyNamespace, fullname, storeReference)
 	logrus.Infof("store '%v' added to store map", storeReference.Name())
 
 	return nil

pkg/controllers/store_controller_test.go

@@ -48,15 +48,15 @@ func TestStoreAdd_EmptyParameter(t *testing.T) {
 	if err := storeAddOrReplace(testStoreSpec, "oras"); err != nil {
 		t.Fatalf("storeAddOrReplace() expected no error, actual %v", err)
 	}
-	if StoreMap.GetStoreCount() != 1 {
-		t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 1 {
+		t.Fatalf("Store map expected size 1, actual %v", NamespacedStores.GetStoreCount())
 	}
 }
 
 func TestStoreAdd_WithParameters(t *testing.T) {
 	resetStoreMap()
-	if StoreMap.GetStoreCount() != 0 {
-		t.Fatalf("Store map expected size 0, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 0 {
+		t.Fatalf("Store map expected size 0, actual %v", NamespacedStores.GetStoreCount())
 	}
 	dirPath, err := utils.CreatePlugin(sampleName)
 	if err != nil {
@@ -69,8 +69,8 @@ func TestStoreAdd_WithParameters(t *testing.T) {
 	if err := storeAddOrReplace(testStoreSpec, "testObject"); err != nil {
 		t.Fatalf("storeAddOrReplace() expected no error, actual %v", err)
 	}
-	if StoreMap.GetStoreCount() != 1 {
-		t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 1 {
+		t.Fatalf("Store map expected size 1, actual %v", NamespacedStores.GetStoreCount())
 	}
 }
 
@@ -138,8 +138,8 @@ func TestStore_UpdateAndDelete(t *testing.T) {
 	if err := storeAddOrReplace(testStoreSpec, sampleName); err != nil {
 		t.Fatalf("storeAddOrReplace() expected no error, actual %v", err)
 	}
-	if StoreMap.GetStoreCount() != 1 {
-		t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 1 {
+		t.Fatalf("Store map expected size 1, actual %v", NamespacedStores.GetStoreCount())
 	}
 
 	// modify the Store
@@ -153,19 +153,19 @@ func TestStore_UpdateAndDelete(t *testing.T) {
 	}
 
 	// validate no Store has been added
-	if StoreMap.GetStoreCount() != 1 {
-		t.Fatalf("Store map should be 1 after replacement, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 1 {
+		t.Fatalf("Store map should be 1 after replacement, actual %v", NamespacedStores.GetStoreCount())
 	}
 
-	StoreMap.DeleteStore(constants.EmptyNamespace, sampleName)
+	NamespacedStores.DeleteStore(constants.EmptyNamespace, sampleName)
 
-	if StoreMap.GetStoreCount() != 0 {
-		t.Fatalf("Store map should be 0 after deletion, actual %v", StoreMap.GetStoreCount())
+	if NamespacedStores.GetStoreCount() != 0 {
+		t.Fatalf("Store map should be 0 after deletion, actual %v", NamespacedStores.GetStoreCount())
 	}
 }
 
 func resetStoreMap() {
-	StoreMap = rs.NewActiveStores()
+	NamespacedStores = rs.NewActiveStores()
 }
 
 func getOrasStoreSpec(pluginName, pluginPath string) configv1beta1.StoreSpec {

pkg/controllers/utils/cert_store.go

@@ -0,0 +1,28 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"crypto/x509"
+
+	ctxUtils "github.com/deislabs/ratify/internal/context"
+	"github.com/deislabs/ratify/pkg/controllers"
+)
+
+// returns the internal certificate map
+// TODO: returns certificates from both cluster-wide and given namespace as namespaced verifier could access both.
+func GetCertificatesMap(ctx context.Context) map[string][]*x509.Certificate {
+	return controllers.NamespacedCertStores.GetCertStores(ctxUtils.GetNamespace(ctx))
+}

pkg/controllers/utils/cert_store_test.go

@@ -0,0 +1,35 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"crypto/x509"
+	"testing"
+
+	"github.com/deislabs/ratify/pkg/controllers"
+
+	ctxUtils "github.com/deislabs/ratify/internal/context"
+	cs "github.com/deislabs/ratify/pkg/customresources/certificatestores"
+)
+
+func TestGetCertificatesMap(t *testing.T) {
+	controllers.NamespacedCertStores = cs.NewActiveCertStores()
+	controllers.NamespacedCertStores.AddStore("default", "default/certStore", []*x509.Certificate{})
+	ctx := ctxUtils.SetContextWithNamespace(context.Background(), "default")
+
+	if certs := GetCertificatesMap(ctx); len(certs) != 1 {
+		t.Fatalf("Expected 1 certificate store, got %d", len(certs))
+	}
+}

pkg/controllers/verifier_controller.go

@@ -68,7 +68,7 @@ func (r *VerifierReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		if apierrors.IsNotFound(err) {
 			verifierLogger.Infof("delete event detected, removing verifier %v", resource)
 			// TODO: pass the actual namespace once multi-tenancy is supported.
-			VerifierMap.DeleteVerifier(constants.EmptyNamespace, resource)
+			NamespacedVerifiers.DeleteVerifier(constants.EmptyNamespace, resource)
 		} else {
 			verifierLogger.Error(err, "unable to fetch verifier")
 		}
@@ -119,7 +119,7 @@ func verifierAddOrReplace(spec configv1beta1.VerifierSpec, objectName string, na
 		return err
 	}
 	// TODO: pass the actual namespace once multi-tenancy is supported.
-	VerifierMap.AddVerifier(constants.EmptyNamespace, objectName, referenceVerifier)
+	NamespacedVerifiers.AddVerifier(constants.EmptyNamespace, objectName, referenceVerifier)
 	logrus.Infof("verifier '%v' added to verifier map", referenceVerifier.Name())
 
 	return nil

pkg/controllers/verifier_controller_test.go

@@ -34,7 +34,7 @@ const licenseChecker = "licensechecker"
 
 func TestMain(m *testing.M) {
 	// make sure to reset verifierMap before each test run
-	VerifierMap = verifiers.NewActiveVerifiers()
+	NamespacedVerifiers = verifiers.NewActiveVerifiers()
 	code := m.Run()
 	os.Exit(code)
 }
@@ -56,15 +56,15 @@ func TestVerifierAdd_EmptyParameter(t *testing.T) {
 	if err := verifierAddOrReplace(testVerifierSpec, sampleName, constants.EmptyNamespace); err != nil {
 		t.Fatalf("verifierAddOrReplace() expected no error, actual %v", err)
 	}
-	if VerifierMap.GetVerifierCount() != 1 {
-		t.Fatalf("Verifier map expected size 1, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 1 {
+		t.Fatalf("Verifier map expected size 1, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 }
 
 func TestVerifierAdd_WithParameters(t *testing.T) {
 	resetVerifierMap()
-	if VerifierMap.GetVerifierCount() != 0 {
-		t.Fatalf("Verifier map expected size 0, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 0 {
+		t.Fatalf("Verifier map expected size 0, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 
 	dirPath, err := utils.CreatePlugin(licenseChecker)
@@ -78,8 +78,8 @@ func TestVerifierAdd_WithParameters(t *testing.T) {
 	if err := verifierAddOrReplace(testVerifierSpec, "testObject", constants.EmptyNamespace); err != nil {
 		t.Fatalf("verifierAddOrReplace() expected no error, actual %v", err)
 	}
-	if VerifierMap.GetVerifierCount() != 1 {
-		t.Fatalf("Verifier map expected size 1, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 1 {
+		t.Fatalf("Verifier map expected size 1, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 }
 
@@ -109,8 +109,8 @@ func TestVerifier_UpdateAndDelete(t *testing.T) {
 	if err := verifierAddOrReplace(testVerifierSpec, licenseChecker, constants.EmptyNamespace); err != nil {
 		t.Fatalf("verifierAddOrReplace() expected no error, actual %v", err)
 	}
-	if VerifierMap.GetVerifierCount() != 1 {
-		t.Fatalf("Verifier map expected size 1, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 1 {
+		t.Fatalf("Verifier map expected size 1, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 
 	// modify the verifier
@@ -121,14 +121,14 @@ func TestVerifier_UpdateAndDelete(t *testing.T) {
 	}
 
 	// validate no verifier has been added
-	if VerifierMap.GetVerifierCount() != 1 {
-		t.Fatalf("Verifier map should be 1 after replacement, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 1 {
+		t.Fatalf("Verifier map should be 1 after replacement, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 
-	VerifierMap.DeleteVerifier(constants.EmptyNamespace, licenseChecker)
+	NamespacedVerifiers.DeleteVerifier(constants.EmptyNamespace, licenseChecker)
 
-	if VerifierMap.GetVerifierCount() != 0 {
-		t.Fatalf("Verifier map should be 0 after deletion, actual %v", VerifierMap.GetVerifierCount())
+	if NamespacedVerifiers.GetVerifierCount() != 0 {
+		t.Fatalf("Verifier map should be 0 after deletion, actual %v", NamespacedVerifiers.GetVerifierCount())
 	}
 }
 
@@ -206,7 +206,7 @@ func TestGetCertStoreNamespace(t *testing.T) {
 }
 
 func resetVerifierMap() {
-	VerifierMap = verifiers.NewActiveVerifiers()
+	NamespacedVerifiers = verifiers.NewActiveVerifiers()
 }
 
 func getLicenseCheckerFromParam(parametersString, pluginPath string) configv1beta1.VerifierSpec {

pkg/customresources/certificatestores/api.go

@@ -0,0 +1,31 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certificatestores
+
+import "crypto/x509"
+
+// CertStoreManager is an interface that defines the methods for managing certificate stores across different scopes.
+type CertStoreManager interface {
+	// GetCertStores returns certificates for the given scope.
+	GetCertStores(scope string) map[string][]*x509.Certificate
+
+	// AddStore adds the given certificate under the given scope.
+	AddStore(scope, storeName string, cert []*x509.Certificate)
+
+	// DeleteStore deletes the certificate from the given scope.
+	DeleteStore(scope, storeName string)
+
+	// IsEmpty returns true if there are no certificates.
+	IsEmpty() bool
+}