From 7e18a20d2983f4ad777214a3087a041c3cc28920 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Thu, 25 Jul 2024 11:57:20 +0300 Subject: [PATCH] operands/kubevirt: Set Kubevirt CR new primary UDN Binding plugin The ability to register network bindings to Kubevirt VMs was proposed [0] in order to allow Kubevirt VMs to connect to network bindings and gain advanced user defined network capabilities. The configuration is added to the kubevirt CR (kubevirt feature gate* + configuring the binding parameters). This commit deploys and registers a primary user-defined-network (UDN) binding to the kubevirt CR, so it could be used by kubevirt users. * enabling the `NetworkBindingPlugins` kubevirt feature-gate on kubevirt CR is not needed as it is already set by default by HCO [1]. [0] https://github.com/kubevirt/community/pull/280 [1] https://github.com/kubevirt/hyperconverged-cluster-operator/pull/2603 Signed-off-by: Ram Lavi --- controllers/operands/kubevirt.go | 46 ++++++++++++++++++-- controllers/operands/kubevirt_test.go | 61 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/controllers/operands/kubevirt.go b/controllers/operands/kubevirt.go index 3fb564df1..1a45e6efe 100644 --- a/controllers/operands/kubevirt.go +++ b/controllers/operands/kubevirt.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "os" + "path" "reflect" "strconv" "strings" @@ -45,6 +46,12 @@ const ( DefaultARM64EmulatedMachines = "virt*" ) +const ( + // Needs to align with the NAD that will be deployed by CNAO + primaryUDNNetworkBindingName = "primary-user-defined-network" + primaryUDNNetworkBindingNamespace = "default" +) + var ( useKVMEmulation = false ) @@ -421,11 +428,13 @@ func getKVConfig(hc *hcov1beta1.HyperConverged) (*kubevirtcorev1.KubeVirtConfigu seccompConfig := getKVSeccompConfig() + networkBindings := getNetworkBindings(hc.Spec.NetworkBinding, hc.Spec.FeatureGates) + config := &kubevirtcorev1.KubeVirtConfiguration{ DeveloperConfiguration: devConfig, NetworkConfiguration: &kubevirtcorev1.NetworkConfiguration{ NetworkInterface: string(kubevirtcorev1.MasqueradeInterface), - Binding: hc.Spec.NetworkBinding, + Binding: networkBindings, }, MigrationConfiguration: kvLiveMigration, PermittedHostDevices: toKvPermittedHostDevices(hc.Spec.PermittedHostDevices), @@ -509,10 +518,26 @@ func getKVConfig(hc *hcov1beta1.HyperConverged) (*kubevirtcorev1.KubeVirtConfigu if hc.Spec.ResourceRequirements != nil { config.AutoCPULimitNamespaceLabelSelector = hc.Spec.ResourceRequirements.AutoCPULimitNamespaceLabelSelector.DeepCopy() } - return config, nil } +func getNetworkBindings( + hcoNetworkBindings map[string]kubevirtcorev1.InterfaceBindingPlugin, + hcoFeatureGates hcov1beta1.HyperConvergedFeatureGates) map[string]kubevirtcorev1.InterfaceBindingPlugin { + networkBindings := maps.Clone(hcoNetworkBindings) + + if hcoFeatureGates.PrimaryUserDefinedNetworkBinding != nil && *hcoFeatureGates.PrimaryUserDefinedNetworkBinding { + if networkBindings == nil { + networkBindings = make(map[string]kubevirtcorev1.InterfaceBindingPlugin) + } + + sidecarImage, _ := os.LookupEnv(hcoutil.PrimaryUDNImageEnvV) + networkBindings[primaryUDNNetworkBindingName] = primaryUserDefinedNetworkBinding(sidecarImage) + } + return networkBindings + +} + func getObsoleteCPUConfig(hcObsoleteCPUConf *hcov1beta1.HyperConvergedObsoleteCPUs) (map[string]bool, string) { obsoleteCPUModels := make(map[string]bool) for _, cpu := range hardcodedObsoleteCPUModels { @@ -736,6 +761,22 @@ func getKVDevConfig(hc *hcov1beta1.HyperConverged) *kubevirtcorev1.DeveloperConf return devConf } +func primaryUserDefinedNetworkBinding(sidecarImage string) kubevirtcorev1.InterfaceBindingPlugin { + const bindingComputeMemoryOverhead = "500Mi" + return kubevirtcorev1.InterfaceBindingPlugin{ + NetworkAttachmentDefinition: path.Join(primaryUDNNetworkBindingNamespace, primaryUDNNetworkBindingName), + SidecarImage: sidecarImage, + Migration: &kubevirtcorev1.InterfaceBindingMigration{ + Method: kubevirtcorev1.LinkRefresh, + }, + ComputeResourceOverhead: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(bindingComputeMemoryOverhead), + }, + }, + } +} + // Static for now, could be configured in the HCO CR in the future func getKVSeccompConfig() *kubevirtcorev1.SeccompConfiguration { return &kubevirtcorev1.SeccompConfiguration{ @@ -811,7 +852,6 @@ func getFeatureGateChecks(featureGates *hcov1beta1.HyperConvergedFeatureGates) [ if featureGates.AutoResourceLimits != nil && *featureGates.AutoResourceLimits { fgs = append(fgs, kvAutoResourceLimits) } - if featureGates.AlignCPUs != nil && *featureGates.AlignCPUs { fgs = append(fgs, kvAlignCPUs) } diff --git a/controllers/operands/kubevirt_test.go b/controllers/operands/kubevirt_test.go index ae22a5651..675a61db4 100644 --- a/controllers/operands/kubevirt_test.go +++ b/controllers/operands/kubevirt_test.go @@ -1815,6 +1815,7 @@ Version: 1.2.3`) Context("Feature Gates", func() { getClusterInfo := hcoutil.GetClusterInfo + const expectedPrimaryUDNImage = "quay.io/some-org/some-repo@some-sha" BeforeEach(func() { hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo { @@ -1826,6 +1827,14 @@ Version: 1.2.3`) hcoutil.GetClusterInfo = getClusterInfo }) + BeforeEach(func() { + Expect(os.Setenv(hcoutil.PrimaryUDNImageEnvV, expectedPrimaryUDNImage)).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv(hcoutil.PrimaryUDNImageEnvV)).To(Succeed()) + }) + Context("test feature gates in NewKubeVirt", func() { It("should add the WithHostPassthroughCPU feature gate if it's set in HyperConverged CR", func() { // one enabled, one disabled and one missing @@ -2056,6 +2065,58 @@ Version: 1.2.3`) Expect(existingResource.Annotations).ToNot(HaveKey(kubevirtcorev1.EmulatorThreadCompleteToEvenParity)) }) + It("should add the Primary User Defined Network Binding to Kubevirt CR if PrimaryUserDefinedNetworkBinding is true in HyperConverged CR", func() { + hco.Spec.FeatureGates = hcov1beta1.HyperConvergedFeatureGates{ + PrimaryUserDefinedNetworkBinding: ptr.To(true), + } + hco.Spec.NetworkBinding = nil + + existingResource, err := NewKubeVirt(hco) + Expect(err).ToNot(HaveOccurred()) + + Expect(existingResource.Spec.Configuration.NetworkConfiguration).NotTo(BeNil()) + Expect(existingResource.Spec.Configuration.NetworkConfiguration.Binding).NotTo(BeNil()) + expectedInterfaceBindingPlugin := kubevirtcorev1.InterfaceBindingPlugin{ + NetworkAttachmentDefinition: primaryUDNNetworkBindingNamespace + "/" + primaryUDNNetworkBindingName, + SidecarImage: expectedPrimaryUDNImage, + Migration: &kubevirtcorev1.InterfaceBindingMigration{ + Method: kubevirtcorev1.LinkRefresh, + }, + ComputeResourceOverhead: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + } + Expect(existingResource.Spec.Configuration.NetworkConfiguration.Binding[primaryUDNNetworkBindingName]).To(Equal(expectedInterfaceBindingPlugin)) + }) + + It("should not add the Primary User Defined Network Binding to Kubevirt CR if PrimaryUserDefinedNetworkBinding is false in HyperConverged CR", func() { + hco.Spec.FeatureGates = hcov1beta1.HyperConvergedFeatureGates{ + PrimaryUserDefinedNetworkBinding: ptr.To(false), + } + hco.Spec.NetworkBinding = nil + + existingResource, err := NewKubeVirt(hco) + Expect(err).ToNot(HaveOccurred()) + + Expect(existingResource.Spec.Configuration.NetworkConfiguration).NotTo(BeNil()) + Expect(existingResource.Spec.Configuration.NetworkConfiguration.Binding).To(BeNil()) + }) + + It("should not add the Primary User Defined Network Binding to Kubevirt CR if PrimaryUserDefinedNetworkBinding is not set in HyperConverged CR", func() { + hco.Spec.FeatureGates = hcov1beta1.HyperConvergedFeatureGates{ + PrimaryUserDefinedNetworkBinding: nil, + } + hco.Spec.NetworkBinding = nil + + existingResource, err := NewKubeVirt(hco) + Expect(err).ToNot(HaveOccurred()) + + Expect(existingResource.Spec.Configuration.NetworkConfiguration).NotTo(BeNil()) + Expect(existingResource.Spec.Configuration.NetworkConfiguration.Binding).To(BeNil()) + }) + It("should not add the feature gates if FeatureGates field is empty", func() { mandatoryKvFeatureGates = getMandatoryKvFeatureGates(false) hco.Spec.FeatureGates = hcov1beta1.HyperConvergedFeatureGates{}