Add oidc certificate and configs for queryserver (#3599)

pkg/controller/apiserver/apiserver_controller.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/tigera/operator/pkg/render/common/authentication"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -292,8 +293,10 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 	var applicationLayer *operatorv1.ApplicationLayer
 	var managementCluster *operatorv1.ManagementCluster
 	var managementClusterConnection *operatorv1.ManagementClusterConnection
+	var keyValidatorConfig authentication.KeyValidatorConfig
 	includeV3NetworkPolicy := false
 	if installationSpec.Variant == operatorv1.TigeraSecureEnterprise {
+		trustedBundle = certificateManager.CreateTrustedBundle()
 		applicationLayer, err = utils.GetApplicationLayer(ctx, r.client)
 		if err != nil {
 			r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger)
@@ -360,7 +363,44 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 			r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger)
 			return reconcile.Result{}, err
 		} else if prometheusCertificate != nil {
-			trustedBundle = certificateManager.CreateTrustedBundle(prometheusCertificate)
+			trustedBundle.AddCertificates(prometheusCertificate)
+		}
+
+		var authenticationCR *operatorv1.Authentication
+		// Fetch the Authentication spec. If present, we use it to configure user authentication.
+		authenticationCR, err = utils.GetAuthentication(ctx, r.client)
+		if err != nil && !errors.IsNotFound(err) {
+			r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger)
+			return reconcile.Result{}, err
+		}
+
+		if authenticationCR != nil && authenticationCR.Status.State != operatorv1.TigeraStatusReady {
+			r.status.SetDegraded(operatorv1.ResourceNotReady,
+				fmt.Sprintf("Authentication is not ready authenticationCR status: %s", authenticationCR.Status.State),
+				nil, reqLogger)
+			return reconcile.Result{}, nil
+		} else if authenticationCR != nil && !utils.IsDexDisabled(authenticationCR) {
+			// Do not include DEX TLS Secret Name if authentication CR does not have type Dex
+			secret := render.DexTLSSecretName
+			certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace())
+			if err != nil {
+				r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret),
+					err, reqLogger)
+				return reconcile.Result{}, err
+			} else if certificate == nil {
+				reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret))
+				r.status.SetDegraded(operatorv1.ResourceNotReady,
+					fmt.Sprintf("Waiting for secret '%s' to become available", secret),
+					nil, reqLogger)
+				return reconcile.Result{}, nil
+			}
+			trustedBundle.AddCertificates(certificate)
+		}
+
+		keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.clusterDomain)
+		if err != nil {
+			r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger)
+			return reconcile.Result{}, err
 		}
 	}
 
@@ -395,6 +435,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 		OpenShift:                   r.provider.IsOpenShift(),
 		TrustedBundle:               trustedBundle,
 		MultiTenant:                 r.multiTenant,
+		KeyValidatorConfig:          keyValidatorConfig,
 	}
 
 	component, err := render.APIServer(&apiServerCfg)

pkg/controller/apiserver/apiserver_controller_test.go

@@ -123,7 +123,11 @@ var _ = Describe("apiserver controller tests", func() {
 					IssuerURL: "https://localhost:9443/dex",
 				},
 			},
+			Status: operatorv1.AuthenticationStatus{
+				State: "Ready",
+			},
 		})).ToNot(HaveOccurred())
+
 		dexSecret, err := secret.CreateTLSSecret(cryptoCA, render.DexTLSSecretName, common.OperatorNamespace(), corev1.TLSPrivateKeyKey, corev1.TLSCertKey, time.Hour, nil, dns.GetServiceDNSNames(render.DexTLSSecretName, render.DexNamespace, dns.DefaultClusterDomain)...)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(cli.Create(ctx, dexSecret)).ToNot(HaveOccurred())
@@ -149,6 +153,9 @@ var _ = Describe("apiserver controller tests", func() {
 		mockStatus.On("RemoveCertificateSigningRequests", mock.Anything)
 		mockStatus.On("ReadyToMonitor")
 		mockStatus.On("SetMetaData", mock.Anything).Return()
+		mockStatus.On("SetDegraded", operatorv1.ResourceReadError, mock.Anything, mock.Anything, mock.Anything).Return().Maybe()
+		mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, mock.Anything, mock.Anything, mock.Anything).Return().Maybe()
+
 	})
 
 	Context("verify reconciliation", func() {

pkg/render/apiserver.go

@@ -16,8 +16,12 @@ package render
 
 import (
 	"fmt"
+	"net/url"
 	"strings"
 
+	v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
+	"github.com/tigera/api/pkg/lib/numorstring"
+	"github.com/tigera/operator/pkg/render/common/authentication"
 	admregv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -29,9 +33,6 @@ import (
 	apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
-	"github.com/tigera/api/pkg/lib/numorstring"
-
 	operatorv1 "github.com/tigera/operator/api/v1"
 	"github.com/tigera/operator/pkg/common"
 	"github.com/tigera/operator/pkg/components"
@@ -126,6 +127,7 @@ type APIServerConfiguration struct {
 	OpenShift                   bool
 	TrustedBundle               certificatemanagement.TrustedBundle
 	MultiTenant                 bool
+	KeyValidatorConfig          authentication.KeyValidatorConfig
 }
 
 type apiServerComponent struct {
@@ -497,11 +499,19 @@ func allowTigeraAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy {
 			Destination: networkpolicy.PrometheusEntityRule,
 		},
 		{
-			// Pass to subsequent tiers for further enforcement
-			Action: v3.Pass,
+			Action:      v3.Allow,
+			Protocol:    &networkpolicy.TCPProtocol,
+			Destination: DexEntityRule,
 		},
 	}...)
 
+	if cfg.KeyValidatorConfig != nil {
+		if parsedURL, err := url.Parse(cfg.KeyValidatorConfig.Issuer()); err == nil {
+			oidcEgressRule := networkpolicy.GetOIDCEgressRule(parsedURL)
+			egressRules = append(egressRules, oidcEgressRule)
+		}
+	}
+
 	if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil {
 		egressRules = append(egressRules, v3.Rule{
 			Action:      v3.Allow,
@@ -510,6 +520,12 @@ func allowTigeraAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy {
 		})
 	}
 
+	// add pass after all egress rules
+	egressRules = append(egressRules, v3.Rule{
+		// Pass to subsequent tiers for further enforcement
+		Action: v3.Pass,
+	})
+
 	// The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on.
 	ingressPorts := networkpolicy.Ports(443, APIServerPort, QueryServerPort, 10443)
 	if cfg.IsSidecarInjectionEnabled() {
@@ -1254,6 +1270,10 @@ func (c *apiServerComponent) queryServerContainer() corev1.Container {
 		env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()})
 	}
 
+	if c.cfg.KeyValidatorConfig != nil {
+		env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...)
+	}
+
 	volumeMounts := []corev1.VolumeMount{
 		c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()),
 	}

pkg/render/common/networkpolicy/networkpolicy.go

@@ -16,6 +16,7 @@ package networkpolicy
 
 import (
 	"fmt"
+	"net/url"
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -92,6 +93,22 @@ func CreateSourceEntityRule(namespace string, deploymentName string) v3.EntityRu
 	}
 }
 
+// GetOIDCEgressRule creates egress rule for oidc connection.
+// the result will include an egress rules with the urlString passed in:
+//  1. egress rule: egress rule assuming the oidc is external to the cluster
+func GetOIDCEgressRule(parsedURL *url.URL) v3.Rule {
+	hostname := parsedURL.Hostname()
+	OIDCEntityRuleExternal := v3.EntityRule{
+		Domains: []string{hostname},
+	}
+
+	return v3.Rule{
+		Action:      v3.Allow,
+		Protocol:    &TCPProtocol,
+		Destination: OIDCEntityRuleExternal,
+	}
+}
+
 // AppendServiceSelectorDNSEgressRules is equivalent to AppendDNSEgressRules, utilizing service selector instead of label selector and ports.
 func AppendServiceSelectorDNSEgressRules(egressRules []v3.Rule, openShift bool) []v3.Rule {
 	if openShift {

pkg/render/compliance.go

@@ -17,6 +17,7 @@ package render
 import (
 	"crypto/x509"
 	"fmt"
+	"net/url"
 	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -1680,6 +1681,13 @@ func (c *complianceComponent) complianceServerAllowTigeraNetworkPolicy() *v3.Net
 
 	egressRules = networkpolicy.AppendDNSEgressRules(egressRules, c.cfg.OpenShift)
 
+	// add oidc egress rule
+	if c.cfg.KeyValidatorConfig != nil {
+		if parsedURL, err := url.Parse(c.cfg.KeyValidatorConfig.Issuer()); err == nil {
+			egressRules = append(egressRules, networkpolicy.GetOIDCEgressRule(parsedURL))
+		}
+	}
+
 	egressRules = append(egressRules, []v3.Rule{
 		{
 			Action:      v3.Allow,

pkg/render/testutils/expected_policies/apiserver.json

@@ -85,6 +85,17 @@
           ]
         }
       },
+      {
+        "action": "Allow",
+        "protocol": "TCP",
+        "destination": {
+          "selector": "k8s-app == 'tigera-dex'",
+          "namespaceSelector": "projectcalico.org/name == 'tigera-dex'",
+          "ports" : [
+            5556
+          ]
+        }
+      },
       {
         "action": "Pass"
       }

pkg/render/testutils/expected_policies/apiserver_ocp.json

@@ -96,6 +96,17 @@
           ]
         }
       },
+      {
+        "action": "Allow",
+        "protocol": "TCP",
+        "destination": {
+          "selector": "k8s-app == 'tigera-dex'",
+          "namespaceSelector": "projectcalico.org/name == 'tigera-dex'",
+          "ports" : [
+            5556
+          ]
+        }
+      },
       {
         "action": "Pass"
       }